RAG-Retriever.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/finardi/tutos/blob/master/RAG-Retriever.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YNbFOEtDTnKL"
      },
      "source": [
        "## Mount Google Drive"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jydgbyHSbtIl",
        "outputId": "ffeba244-d966-4e4b-e0f9-d3b12cc3a771"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Install Libs"
      ],
      "metadata": {
        "id": "bdau-U9CHM8B"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XPF1_OOrRYaj",
        "outputId": "1277f50d-1e83-4611-cd29-4c2ebdb0d9a1"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": []
          },
          "metadata": {},
          "execution_count": 2
        }
      ],
      "source": [
        "%%shell\n",
        "pip install faiss-cpu -q\n",
        "pip install pyserini==0.25.0 -q\n",
        "pip install ftfy -q"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qUABwgO1Rbtd",
        "outputId": "768fb231-5295-411d-d1c2-b5426cc83ccd"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "fatal: destination path 'pyserini' already exists and is not an empty directory.\n",
            "trec_eval.9.0.4/\n",
            "trec_eval.9.0.4/m_prefs_pair.c\n",
            "trec_eval.9.0.4/m_ndcg_p.c\n",
            "trec_eval.9.0.4/m_infap.c\n",
            "trec_eval.9.0.4/m_num_q.c\n",
            "trec_eval.9.0.4/m_iprec_at_recall.c\n",
            "trec_eval.9.0.4/form_prefs_counts.c\n",
            "trec_eval.9.0.4/m_prefs_num_prefs_ful_ret.c\n",
            "trec_eval.9.0.4/utility_pool.c\n",
            "trec_eval.9.0.4/m_binG.c\n",
            "trec_eval.9.0.4/meas_avg.c\n",
            "trec_eval.9.0.4/m_gm_bpref.c\n",
            "trec_eval.9.0.4/m_runid.c\n",
            "trec_eval.9.0.4/m_bpref.c\n",
            "trec_eval.9.0.4/m_gm_map.c\n",
            "trec_eval.9.0.4/trec_eval.h\n",
            "trec_eval.9.0.4/m_yaap.c\n",
            "trec_eval.9.0.4/m_relstring.c\n",
            "trec_eval.9.0.4/m_Rprec.c\n",
            "trec_eval.9.0.4/m_prefs_avgjg.c\n",
            "trec_eval.9.0.4/m_success.c\n",
            "trec_eval.9.0.4/m_ndcg.c\n",
            "trec_eval.9.0.4/functions.h\n",
            "trec_eval.9.0.4/m_P_avgjg.c\n",
            "trec_eval.9.0.4/test/\n",
            "trec_eval.9.0.4/test/qrels.rel_level\n",
            "trec_eval.9.0.4/test/results.test\n",
            "trec_eval.9.0.4/test/qrels.test\n",
            "trec_eval.9.0.4/test/out.test.qrels_jg\n",
            "trec_eval.9.0.4/test/out.test.meas_params\n",
            "trec_eval.9.0.4/test/out.test.a\n",
            "trec_eval.9.0.4/test/out.test.prefs\n",
            "trec_eval.9.0.4/test/out.test.aqcM\n",
            "trec_eval.9.0.4/test/out.test.aql\n",
            "trec_eval.9.0.4/test/prefs.test\n",
            "trec_eval.9.0.4/test/out.test\n",
            "trec_eval.9.0.4/test/out.test.aq\n",
            "trec_eval.9.0.4/test/out.test.aqc\n",
            "trec_eval.9.0.4/test/out.test.qrels_prefs\n",
            "trec_eval.9.0.4/test/zscores_file\n",
            "trec_eval.9.0.4/test/qrels.123\n",
            "trec_eval.9.0.4/test/out.test.aqZ\n",
            "trec_eval.9.0.4/test/results.trunc\n",
            "trec_eval.9.0.4/test/prefs.results.test\n",
            "trec_eval.9.0.4/test/prefs.rank20\n",
            "trec_eval.9.0.4/m_11pt_avg.c\n",
            "trec_eval.9.0.4/m_G.c\n",
            "trec_eval.9.0.4/m_num_rel.c\n",
            "trec_eval.9.0.4/m_map_cut.c\n",
            "trec_eval.9.0.4/m_prefs_avgjg_ret.c\n",
            "trec_eval.9.0.4/m_Rprec_mult.c\n",
            "trec_eval.9.0.4/Makefile\n",
            "trec_eval.9.0.4/m_map_avgjg.c\n",
            "trec_eval.9.0.4/get_qrels_prefs.c\n",
            "trec_eval.9.0.4/README\n",
            "trec_eval.9.0.4/m_set_rel_P.c\n",
            "trec_eval.9.0.4/sysfunc.h\n",
            "trec_eval.9.0.4/m_prefs_pair_ret.c\n",
            "trec_eval.9.0.4/convert_zscores.c\n",
            "trec_eval.9.0.4/m_ndcg_cut.c\n",
            "trec_eval.9.0.4/m_prefs_pair_imp.c\n",
            "trec_eval.9.0.4/meas_print_single.c\n",
            "trec_eval.9.0.4/meas_print_final.c\n",
            "trec_eval.9.0.4/trec_eval.c\n",
            "trec_eval.9.0.4/m_num_ret.c\n",
            "trec_eval.9.0.4/get_prefs.c\n",
            "trec_eval.9.0.4/m_P.c\n",
            "trec_eval.9.0.4/get_qrels_jg.c\n",
            "trec_eval.9.0.4/m_rel_P.c\n",
            "trec_eval.9.0.4/meas_acc.c\n",
            "trec_eval.9.0.4/m_prefs_simp.c\n",
            "trec_eval.9.0.4/m_recall.c\n",
            "trec_eval.9.0.4/trec_format.h\n",
            "trec_eval.9.0.4/m_ndcg_rel.c\n",
            "trec_eval.9.0.4/m_num_nonrel_judged_ret.c\n",
            "trec_eval.9.0.4/formats.c\n",
            "trec_eval.9.0.4/bpref_bug\n",
            "trec_eval.9.0.4/README.windows.md\n",
            "trec_eval.9.0.4/m_prefs_num_prefs_ful.c\n",
            "trec_eval.9.0.4/m_set_map.c\n",
            "trec_eval.9.0.4/get_qrels.c\n",
            "trec_eval.9.0.4/m_set_F.c\n",
            "trec_eval.9.0.4/measures.c\n",
            "trec_eval.9.0.4/common.h\n",
            "trec_eval.9.0.4/meas_init.c\n",
            "trec_eval.9.0.4/m_recip_rank.c\n",
            "trec_eval.9.0.4/m_set_recall.c\n",
            "trec_eval.9.0.4/get_trec_results.c\n",
            "trec_eval.9.0.4/m_prefs_simp_ret.c\n",
            "trec_eval.9.0.4/m_num_rel_ret.c\n",
            "trec_eval.9.0.4/m_map.c\n",
            "trec_eval.9.0.4/m_utility.c\n",
            "trec_eval.9.0.4/form_res_rels.c\n",
            "trec_eval.9.0.4/form_res_rels_jg.c\n",
            "trec_eval.9.0.4/m_prefs_simp_imp.c\n",
            "trec_eval.9.0.4/m_prefs_num_prefs_poss.c\n",
            "trec_eval.9.0.4/m_prefs_avgjg_Rnonrel.c\n",
            "trec_eval.9.0.4/m_prefs_avgjg_Rnonrel_ret.c\n",
            "trec_eval.9.0.4/m_set_P.c\n",
            "trec_eval.9.0.4/m_prefs_avgjg_imp.c\n",
            "trec_eval.9.0.4/m_Rndcg.c\n",
            "trec_eval.9.0.4/CHANGELOG\n",
            "trec_eval.9.0.4/m_Rprec_mult_avgjg.c\n",
            "trec_eval.9.0.4/get_zscores.c\n",
            "make: 'trec_eval' is up to date.\n",
            "make: 'ndeval' is up to date.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": []
          },
          "metadata": {},
          "execution_count": 3
        }
      ],
      "source": [
        "%%shell\n",
        "apt-get install maven -qq\n",
        "git clone --recurse-submodules https://github.com/castorini/pyserini.git\n",
        "cd pyserini\n",
        "cd tools/eval && tar xvfz trec_eval.9.0.4.tar.gz && cd trec_eval.9.0.4 && make && cd ../../..\n",
        "cd tools/eval/ndeval && make && cd ../../.."
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Import Libs"
      ],
      "metadata": {
        "id": "1AcoIPKpHQVO"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "aEsUFat2ouIw"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "import re\n",
        "import ftfy\n",
        "import math\n",
        "import json\n",
        "import torch\n",
        "import shutil\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import collections"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from tqdm import tqdm\n",
        "from time import time\n",
        "from IPython.display import display\n",
        "from collections import defaultdict, Counter\n",
        "from pyserini.search.lucene import LuceneSearcher"
      ],
      "metadata": {
        "id": "7g2zNbRRHS3-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import locale\n",
        "locale.getpreferredencoding = lambda: \"UTF-8\""
      ],
      "metadata": {
        "id": "Y2xZAC-x4X_w"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9nabsht5JysO"
      },
      "source": [
        "## Dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JEKaGqglH_FJ"
      },
      "outputs": [],
      "source": [
        "# define o path\n",
        "path = f\"/content/drive/MyDrive/22h/RAG/data/\""
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# recupera os dados\n",
        "shutil.copyfile(os.path.join(path, \"topics.tsv\"), \"/content/topics.tsv\")\n",
        "shutil.copyfile(os.path.join(path, \"corpus_chunk.tsv\"), \"/content/chunk.tsv\")\n",
        "shutil.copyfile(os.path.join(path, \"qrels.tsv\"), \"/content/qrels.tsv\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "aLunKMj3hF4O",
        "outputId": "7ebfb98f-6e13-4783-a656-663dadd6a81b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'/content/qrels.tsv'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SJVYUQUELbYm",
        "outputId": "5ecd7ddf-a273-4f40-b3f8-b505dd08f725"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Reading file on /content/topics.tsv: 140it [00:00, 116485.33it/s]\n"
          ]
        }
      ],
      "source": [
        "# carrega o arquivo dos topics\n",
        "input_tsv = f'/content/topics.tsv'\n",
        "topics = {}\n",
        "with open(input_tsv,'r') as f_in:\n",
        "    for line in tqdm(f_in, desc=f'Reading file on {input_tsv}'):\n",
        "        id, text = line.strip().split('\\t')\n",
        "        topics[int(id)] = text"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "list(topics.items())[:3]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nVKfInBOImAm",
        "outputId": "6536df20-cf2c-4eff-825a-2b7a5ca2d9df"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[(0,\n",
              "  'Qual é o segredo que o Sr. e a Sra. Dursley temiam que alguém descobrisse?'),\n",
              " (1,\n",
              "  'Qual era a cor da capa que o Sr. Dursley viu um homem mais velho do que ele usando, que o fez pensar que poderia ser uma promoção boba?'),\n",
              " (2,\n",
              "  'Qual foi a reação do Sr. Dursley ao ser abraçado por um estranho que usava uma capa roxa e o chamou de trouxa?')]"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "y_WWJYmsUfLd",
        "outputId": "c27126e8-b7a4-44a3-d80c-35ed60df4dba"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Reading file on /content/chunk.tsv: 140it [00:00, 55354.69it/s]\n"
          ]
        }
      ],
      "source": [
        "# carrega o arquivo do corpus\n",
        "input_tsv = f'/content/chunk.tsv'\n",
        "corpus = {}\n",
        "with open(input_tsv,'r') as f_in:\n",
        "    for line in tqdm(f_in, desc=f'Reading file on {input_tsv}'):\n",
        "        id, text = line.strip().split('\\t')\n",
        "        corpus[int(id)] = text"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "list(corpus.items())[:1]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Tuw4KzQ5Iq7N",
        "outputId": "11d0186d-3856-4b5f-8e50-63a2a4b277b7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[(0,\n",
              "  '- CAPÍTULO UM - O menino que sobreviveu O Sr. e a Sra. Dursley, da rua dos Alfeneiros, no 4, se orgulhavam de dizer que eram perfeitamente normais, muito bem, obrigado. Eram as últimas pessoas no mundo que se esperaria que se metessem em alguma coisa estranha ou misteriosa, porque simplesmente não compactuavam com esse tipo de bobagem. O Sr. Dursley era diretor de uma firma chamada Grunnings, que fazia perfurações. Era um homem alto e corpulento quase sem pescoço, embora tivesse enormes bigodes. A Sra. Dursley era magra e loura e tinha um pescoço quase duas vezes mais comprido que o normal, o que era muito útil porque ela passava grande parte do tempo espichando-o por cima da cerca do jardim para espiar os vizinhos. Os Dursley tinham um filhinho chamado Dudley, o Duda, e em sua opinião não havia garoto melhor em nenhum lugar do mundo. Os Dursley tinham tudo que queriam, mas tinham também um segredo, e seu maior receio era que alguém o descobrisse. Achavam que não iriam aguentar se alguém descobrisse a existência dos Potter. A Sra. Potter era irmã da Sra. Dursley, mas não se viam havia muitos anos; na realidade a Sra. Dursley fingia que não tinha irmã, porque esta e o marido imprestável eram o que havia de menos parecido possível com os Dursley. Eles estremeciam só de pensar o que os vizinhos iriam dizer se os Potter aparecessem na rua. Os Dursley sabiam que os Potter tinham um filhinho, também, mas nunca o tinham visto. O garoto era mais uma razão para manter os Potter a distância; eles não queriam que Duda se misturasse com uma criança daquelas. Quando o Sr. e a Sra. Dursley acordaram na terça-feira monótona e cinzenta em que a nossa história começa, não havia nada no céu nublado lá fora sugerindo as coisas estranhas e misteriosas que não tardariam a acontecer por todo o país. O Sr. Dursley cantarolava ao escolher a gravata mais sem graça do mundo para ir trabalhar e a Sra. Dursley fofocava alegremente enquanto lutava  para encaixar um Duda aos berros na cadeirinha alta. Nenhum deles reparou em uma coruja parda que passou, batendo as asas, pela janela. Às oito e meia, o Sr. Dursley apanhou a pasta, deu um beijinho no rosto da Sra. Dursley e tentou dar um beijo de despedida em Duda mas não conseguiu, porque na hora Duda estava tendo um acesso de raiva e atirava o cereal nas paredes. - Pestinha - disse rindo contrafeito o Sr. Dursley ao sair de casa. Entrou no carro e deu marcha à ré para sair do estacionamento do número quatro. Foi na esquina da rua que ele notou o primeiro indício de que algo estranho ocorria - um gato lia um mapa. Por um instante o Sr. Dursley não percebeu o que vira - em seguida virou rapidamente a cabeça para dar uma segunda olhada. Havia um gato de listras amarelas sentado na esquina da rua dos Alfeneiros, mas não havia nenhum mapa à vista. Em que estaria pensando naquela hora? Devia ter sido um efeito da luz. Ele piscou e arregalou os olhos para o gato. O gato o encarou. Enquanto virava a esquina e subia a rua, espiou o gato pelo espelho retrovisor. Ele agora estava lendo a placa que dizia rua dos Alfeneiros - não, estava olhando a placa: gatos não podiam ler mapas nem placas. O Sr. Dursley sacudiu a cabeça e tirou o gato do pensamento. Dur')]"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 423
        },
        "id": "PpOMDrekTIzB",
        "outputId": "515d01a4-3ac4-4838-ffa6-29a0b7f3a6ec"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "     query_id  0  doc_id  rel\n",
              "0           0  0       0    1\n",
              "1           1  0       1    1\n",
              "2           2  0       2    1\n",
              "3           3  0       3    1\n",
              "4           4  0       4    1\n",
              "..        ... ..     ...  ...\n",
              "135       135  0     135    1\n",
              "136       136  0     136    1\n",
              "137       137  0     137    1\n",
              "138       138  0     138    1\n",
              "139       139  0     139    1\n",
              "\n",
              "[140 rows x 4 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-22bcc2cc-e996-460b-9eba-4a88a69b129f\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>query_id</th>\n",
              "      <th>0</th>\n",
              "      <th>doc_id</th>\n",
              "      <th>rel</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>135</th>\n",
              "      <td>135</td>\n",
              "      <td>0</td>\n",
              "      <td>135</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>136</th>\n",
              "      <td>136</td>\n",
              "      <td>0</td>\n",
              "      <td>136</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>137</th>\n",
              "      <td>137</td>\n",
              "      <td>0</td>\n",
              "      <td>137</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>138</th>\n",
              "      <td>138</td>\n",
              "      <td>0</td>\n",
              "      <td>138</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>139</th>\n",
              "      <td>139</td>\n",
              "      <td>0</td>\n",
              "      <td>139</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>140 rows × 4 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-22bcc2cc-e996-460b-9eba-4a88a69b129f')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-22bcc2cc-e996-460b-9eba-4a88a69b129f button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-22bcc2cc-e996-460b-9eba-4a88a69b129f');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-40a177cc-797b-4307-8cc4-d70c90f018a8\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-40a177cc-797b-4307-8cc4-d70c90f018a8')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-40a177cc-797b-4307-8cc4-d70c90f018a8 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "  <div id=\"id_6765d09c-014f-401f-ad8f-e9198b245784\">\n",
              "    <style>\n",
              "      .colab-df-generate {\n",
              "        background-color: #E8F0FE;\n",
              "        border: none;\n",
              "        border-radius: 50%;\n",
              "        cursor: pointer;\n",
              "        display: none;\n",
              "        fill: #1967D2;\n",
              "        height: 32px;\n",
              "        padding: 0 0 0 0;\n",
              "        width: 32px;\n",
              "      }\n",
              "\n",
              "      .colab-df-generate:hover {\n",
              "        background-color: #E2EBFA;\n",
              "        box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "        fill: #174EA6;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate {\n",
              "        background-color: #3B4455;\n",
              "        fill: #D2E3FC;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate:hover {\n",
              "        background-color: #434B5C;\n",
              "        box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "        filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "        fill: #FFFFFF;\n",
              "      }\n",
              "    </style>\n",
              "    <button class=\"colab-df-generate\" onclick=\"generateWithVariable('qrels')\"\n",
              "            title=\"Generate code using this dataframe.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M7,19H8.4L18.45,9,17,7.55,7,17.6ZM5,21V16.75L18.45,3.32a2,2,0,0,1,2.83,0l1.4,1.43a1.91,1.91,0,0,1,.58,1.4,1.91,1.91,0,0,1-.58,1.4L9.25,21ZM18.45,9,17,7.55Zm-12,3A5.31,5.31,0,0,0,4.9,8.1,5.31,5.31,0,0,0,1,6.5,5.31,5.31,0,0,0,4.9,4.9,5.31,5.31,0,0,0,6.5,1,5.31,5.31,0,0,0,8.1,4.9,5.31,5.31,0,0,0,12,6.5,5.46,5.46,0,0,0,6.5,12Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "    <script>\n",
              "      (() => {\n",
              "      const buttonEl =\n",
              "        document.querySelector('#id_6765d09c-014f-401f-ad8f-e9198b245784 button.colab-df-generate');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      buttonEl.onclick = () => {\n",
              "        google.colab.notebook.generateWithVariable('qrels');\n",
              "      }\n",
              "      })();\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "qrels",
              "summary": "{\n  \"name\": \"qrels\",\n  \"rows\": 140,\n  \"fields\": [\n    {\n      \"column\": \"query_id\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 40,\n        \"min\": 0,\n        \"max\": 139,\n        \"num_unique_values\": 140,\n        \"samples\": [\n          108,\n          67,\n          31\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"0\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0,\n        \"min\": 0,\n        \"max\": 0,\n        \"num_unique_values\": 1,\n        \"samples\": [\n          0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"doc_id\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 40,\n        \"min\": 0,\n        \"max\": 139,\n        \"num_unique_values\": 140,\n        \"samples\": [\n          108\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"rel\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0,\n        \"min\": 1,\n        \"max\": 1,\n        \"num_unique_values\": 1,\n        \"samples\": [\n          1\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 13
        }
      ],
      "source": [
        "# carrega o qrels\n",
        "qrels = pd.read_csv(\"/content/qrels.tsv\", sep = \"\\t\", header = None)\n",
        "qrels.columns = [\"query_id\", \"0\", \"doc_id\", \"rel\"]\n",
        "qrels"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "trec_results = {}"
      ],
      "metadata": {
        "id": "yKkVFrU57u_g"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Create Folders"
      ],
      "metadata": {
        "id": "ByBH5PTPJrB4"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!rm -rf runs\n",
        "!rm -rf evals\n",
        "!mkdir runs\n",
        "!mkdir evals"
      ],
      "metadata": {
        "id": "g0ZePSJSJtGU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NRp0VIA3bquq"
      },
      "source": [
        "## Pyserini BM25\n",
        "\n",
        "* https://github.com/castorini/anserini/blob/master/src/main/resources/regression/msmarco-passage.yaml\n",
        "* https://github.com/castorini/pyserini/blob/f75adca8c410e64b3ff1375e181a0ea3af1ddb28/docs/usage-index.md"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "B_2Xm8xgbwl8",
        "outputId": "63490840-9291-45a7-bafe-43855cfc7924"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Writing file on /content/collections/corpus.jsonl: 100%|██████████| 140/140 [00:00<00:00, 17437.34it/s]\n"
          ]
        }
      ],
      "source": [
        "# Construção do índice invertido a partir do corpus\n",
        "!rm -rf /content/collections\n",
        "!mkdir /content/collections\n",
        "\n",
        "# Salva o corpus como um arquivo jsonl\n",
        "output_json = f'/content/collections/corpus.jsonl'\n",
        "\n",
        "with open(output_json,'w') as f_out:\n",
        "    for doc_id, doc_text in tqdm(corpus.items(), desc=f'Writing file on {output_json}'):\n",
        "        output_dict = {'id': doc_id, 'contents': doc_text}\n",
        "        f_out.write(json.dumps(output_dict, ensure_ascii=False) + '\\n')"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!head -n 3 /content/collections/corpus.jsonl"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EGqKApGk7_e7",
        "outputId": "ebe762df-23aa-4aa0-8c03-8cef256fa326"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{\"id\": 0, \"contents\": \"- CAPÍTULO UM - O menino que sobreviveu O Sr. e a Sra. Dursley, da rua dos Alfeneiros, no 4, se orgulhavam de dizer que eram perfeitamente normais, muito bem, obrigado. Eram as últimas pessoas no mundo que se esperaria que se metessem em alguma coisa estranha ou misteriosa, porque simplesmente não compactuavam com esse tipo de bobagem. O Sr. Dursley era diretor de uma firma chamada Grunnings, que fazia perfurações. Era um homem alto e corpulento quase sem pescoço, embora tivesse enormes bigodes. A Sra. Dursley era magra e loura e tinha um pescoço quase duas vezes mais comprido que o normal, o que era muito útil porque ela passava grande parte do tempo espichando-o por cima da cerca do jardim para espiar os vizinhos. Os Dursley tinham um filhinho chamado Dudley, o Duda, e em sua opinião não havia garoto melhor em nenhum lugar do mundo. Os Dursley tinham tudo que queriam, mas tinham também um segredo, e seu maior receio era que alguém o descobrisse. Achavam que não iriam aguentar se alguém descobrisse a existência dos Potter. A Sra. Potter era irmã da Sra. Dursley, mas não se viam havia muitos anos; na realidade a Sra. Dursley fingia que não tinha irmã, porque esta e o marido imprestável eram o que havia de menos parecido possível com os Dursley. Eles estremeciam só de pensar o que os vizinhos iriam dizer se os Potter aparecessem na rua. Os Dursley sabiam que os Potter tinham um filhinho, também, mas nunca o tinham visto. O garoto era mais uma razão para manter os Potter a distância; eles não queriam que Duda se misturasse com uma criança daquelas. Quando o Sr. e a Sra. Dursley acordaram na terça-feira monótona e cinzenta em que a nossa história começa, não havia nada no céu nublado lá fora sugerindo as coisas estranhas e misteriosas que não tardariam a acontecer por todo o país. O Sr. Dursley cantarolava ao escolher a gravata mais sem graça do mundo para ir trabalhar e a Sra. Dursley fofocava alegremente enquanto lutava  para encaixar um Duda aos berros na cadeirinha alta. Nenhum deles reparou em uma coruja parda que passou, batendo as asas, pela janela. Às oito e meia, o Sr. Dursley apanhou a pasta, deu um beijinho no rosto da Sra. Dursley e tentou dar um beijo de despedida em Duda mas não conseguiu, porque na hora Duda estava tendo um acesso de raiva e atirava o cereal nas paredes. - Pestinha - disse rindo contrafeito o Sr. Dursley ao sair de casa. Entrou no carro e deu marcha à ré para sair do estacionamento do número quatro. Foi na esquina da rua que ele notou o primeiro indício de que algo estranho ocorria - um gato lia um mapa. Por um instante o Sr. Dursley não percebeu o que vira - em seguida virou rapidamente a cabeça para dar uma segunda olhada. Havia um gato de listras amarelas sentado na esquina da rua dos Alfeneiros, mas não havia nenhum mapa à vista. Em que estaria pensando naquela hora? Devia ter sido um efeito da luz. Ele piscou e arregalou os olhos para o gato. O gato o encarou. Enquanto virava a esquina e subia a rua, espiou o gato pelo espelho retrovisor. Ele agora estava lendo a placa que dizia rua dos Alfeneiros - não, estava olhando a placa: gatos não podiam ler mapas nem placas. O Sr. Dursley sacudiu a cabeça e tirou o gato do pensamento. Dur\"}\n",
            "{\"id\": 1, \"contents\": \"ante o caminho para a cidade ele não pensou em mais nada exceto no grande pedido de brocas que tinha esperanças de receber naquele dia. Mas ao sair da cidade, as brocas foram varridas de sua cabeça por outra coisa. Ao parar no costumeiro engarrafamento matinal, não pôde deixar de notar que havia uma quantidade de gente estranhamente vestida andando pelas ruas. Gente com capas largas. O Sr. Dursley não tolerava gente que andava com roupas ridículas - os trapos que se viam nos jovens! Imaginou que aquilo fosse uma nova moda idiota. Tamborilou os dedos no volante e seu olhar recaiu em um grupinho de excêntricos parados bem perto dele. Cochichavam excitados. O Sr. Dursley se irritou ao ver que alguns deles nem eram jovens; ora, aquele homem devia ser mais velho do que ele, e usava uma capa verde-esmeralda! Que petulância! Mas então ocorreu ao Sr. Dursley que se tratava provavelmente de alguma promoção boba - essas pessoas estavam obviamente arrecadando alguma coisa... é, devia ser isto! O tráfego avançou e alguns minutos depois o Sr. Dursley chegou ao estacionamento da Grunnings, o pensamento de volta às brocas. O Sr. Dursley sempre sentava de costas para a janela em seu escritório no nono andar. Se não o fizesse, talvez tivesse achado mais difícil se concentrar em brocas aquela manhã. Ele não viu as corujas que voavam velozes em plena luz do dia, embora as pessoas na rua as vissem; elas apontavam e se espantavam  enquanto coruja atrás de coruja passava no alto. A maioria jamais vira uma coruja mesmo à noite. O Sr. Dursley, porém, teve uma manhã perfeitamente normal sem corujas. Gritou com cinco pessoas diferentes. Deu vários telefonemas importantes e gritou mais um pouco. Estava de excelente humor até a hora do almoço, quando pensou em esticar as pernas e atravessar a rua para comprar um pãozinho doce na padaria defronte. Esquecera completamente as pessoas de capas até passar por um grupo delas próximo à padaria. Olhou-as com raiva ao passar. Não sabia o porquê, mas elas o deixavam nervoso. Essas cochichavam agitadas, também, mas ele não viu nenhuma latinha de coleta. Foi ao passar por elas, na volta, levando uma grande rosca açucarada em um saco, que entreouviu algumas palavras do que diziam. - ... Os Potter, é verdade, foi o que ouvi... - ... é, o filho deles, Harry... O Sr. Dursley parou de repente. O medo invadiu-o. Virou a cabeça para olhar as pessoas que cochichavam como se quisesse dizer alguma coisa, mas pensou melhor. Atravessou a rua depressa, correu para o escritório, disse rispidamente à secretária que não o incomodasse, agarrou o telefone e quase terminara de discar o número de casa quando mudou de ideia. Pôs o fone no gancho e alisou os bigodes, pensando... não, estava agindo como um idiota. Potter não era um nome tão fora do comum assim. Tinha certeza de que havia muita gente chamada Potter com um filho chamado Harry. Pensando bem, nem sequer tinha certeza de que o sobrinho tivesse o nome de Harry. Jamais vira o menino. Talvez fosse Ernesto. Ou Eduardo. Não tinha sentido preocupar a Sra. Dursley, ela sempre ficava tão perturbada à simples menção da irmã. Não a culpava - se ele tivesse uma irmã como aquela... mas, mesmo assim, aquelas pessoas de capas... Achou bem mais difícil se concentrar nas brocas aquela tarde e, quando deixou o edifício às cinco horas, continuava tão preocupado que deu um encontrão\"}\n",
            "{\"id\": 2, \"contents\": \" em alguém parado ali à porta. - Desculpe - murmurou, quando o velhinho cambaleou e quase caiu. Levou alguns segundos até o Sr. Dursley perceber que o homem estava usando uma capa roxa. Não parecia nada aborrecido por ter sido quase jogado ao chão. Ao contrário, seu rosto se abriu em um largo sorriso e ele disse numa voz esganiçada que fez os passantes olharem: - Não precisa pedir desculpas, caro senhor, porque nada poderia me aborrecer hoje! Alegre-se, porque Você-Sabe-Quem finalmente foi-se embora! Até trouxas como o senhor deviam estar comemorando um dia tão feliz! E o velho abraçou o Sr. Dursley pela cintura e se afastou. O Sr. Dursley ficou pregado no chão. Fora abraçado por um completo  estranho. E também achava que fora chamado de trouxa, o que quer que isso quisesse dizer. Estava abalado. Correu para o carro e partiu para casa, esperando que estivesse imaginando coisas, o que nunca esperara que fizesse, porque não aprovava a imaginação. Quando entrou no estacionamento do número quatro, a primeira coisa que viu - e isso não melhorou o seu estado de espírito - foi o gato listrado que notara aquela manhã. Agora ele estava sentado no muro do jardim. Tinha certeza de que era o mesmo; as marcas em volta dos olhos eram as mesmas. - Chispa! - disse o Sr. Dursley em voz alta. O gato não se mexeu. Apenas lançou-lhe um olhar severo. Será que isto era um comportamento normal para um gato?, pensou o Sr. Dursley. Continuava decidido a não comentar nada com a esposa. A Sra. Dursley tivera um dia normal e agradável. Contou-lhe durante o jantar os problemas da senhora do lado com a filha e ainda que Duda aprendera uma palavra nova (\\\"Nunca\\\"). O Sr. Dursley tentou agir normalmente. Depois que Duda foi se deitar, ele chegou à sala em tempo de ouvir o último noticiário noturno. \\\"E, por último, os observadores de pássaros em toda parte registraram que as corujas do país se comportaram de forma muito estranha hoje. Embora elas normalmente cacem à noite e raramente apareçam à luz do dia, centenas desses pássaros foram vistos hoje voando em todas as direções desde o alvorecer. Os especialistas não sabem explicar por que as corujas de repente mudaram o seu padrão de sono.\\\" O locutor se permitiu um sorriso. \\\"Muito misterioso. E agora, com Jorge Mendes, o nosso boletim meteorológico. Vai haver mais tempestades de corujas hoje à noite, Jorge?\\\" \\\"Bom, Eduardo\\\", disse o meteorologista, \\\"não sei lhe dizer, mas não foram só as corujas que se comportaram de modo estranho hoje. Ouvintes de todo o país têm telefonado para reclamar que em vez do aguaceiro que prometi para ontem, eles têm tido chuvas de estrelas! Talvez alguém ande festejando a noite das fogueiras uma semana mais cedo este ano! Mas posso prometer para hoje uma noite chuvosa.\\\" O Sr. Dursley ficou paralisado na poltrona. Estrelas cadentes em todo o país? Corujas voando durante o dia? Gente misteriosa usando capas por todo lado? E um cochicho, um cochicho a respeito dos Potter... A Sra. Dursley entrou na sala trazendo duas xícaras de chá. Não adiantava. Teria que lhe dizer alguma coisa. Pigarreou nervoso. - Hum, hum, Petúnia, querida, você não tem tido notícias de sua irmã, ultimamente? Conforme esperava, a Sra. Dursley pareceu chocada e ab\"}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!rm -rf indexes\n",
        "\n",
        "# Cria o índice invertido a partir do arquivo jsonl\n",
        "\"\"\"\n",
        "  língua: pt\n",
        "  stemming: porter\n",
        "  stopwords: remove\n",
        "\"\"\"\n",
        "!python -m pyserini.index.lucene \\\n",
        "--collection JsonCollection \\\n",
        "--input /content/collections \\\n",
        "--index indexes/lucene-index-trec-hp \\\n",
        "--generator DefaultLuceneDocumentGenerator \\\n",
        "--threads 9 \\\n",
        "--language pt \\\n",
        "--storePositions \\\n",
        "--storeDocvectors \\\n",
        "--storeRaw"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "X84Rn-6qucsy",
        "outputId": "79370005-2e49-43f9-b519-6600555e17b4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "WARNING: sun.reflect.Reflection.getCallerClass is not supported. This will impact performance.\n",
            "2024-04-28 20:10:58,751 INFO  [main] index.AbstractIndexer (AbstractIndexer.java:204) - Setting log level to INFO\n",
            "2024-04-28 20:10:58,753 INFO  [main] index.AbstractIndexer (AbstractIndexer.java:207) - ============ Loading Index Configuration ============\n",
            "2024-04-28 20:10:58,754 INFO  [main] index.AbstractIndexer (AbstractIndexer.java:208) - AbstractIndexer settings:\n",
            "2024-04-28 20:10:58,754 INFO  [main] index.AbstractIndexer (AbstractIndexer.java:209) -  + DocumentCollection path: /content/collections\n",
            "2024-04-28 20:10:58,754 INFO  [main] index.AbstractIndexer (AbstractIndexer.java:210) -  + CollectionClass: JsonCollection\n",
            "2024-04-28 20:10:58,755 INFO  [main] index.AbstractIndexer (AbstractIndexer.java:211) -  + Index path: indexes/lucene-index-trec-hp\n",
            "2024-04-28 20:10:58,755 INFO  [main] index.AbstractIndexer (AbstractIndexer.java:212) -  + Threads: 9\n",
            "2024-04-28 20:10:58,756 INFO  [main] index.AbstractIndexer (AbstractIndexer.java:213) -  + Optimize (merge segments)? false\n",
            "2024-04-28 20:10:58,794 INFO  [main] index.IndexCollection (IndexCollection.java:237) - Using language-specific analyzer\n",
            "2024-04-28 20:10:58,795 INFO  [main] index.IndexCollection (IndexCollection.java:238) - Language: pt\n",
            "2024-04-28 20:10:58,933 INFO  [main] index.IndexCollection (IndexCollection.java:197) - IndexCollection settings:\n",
            "2024-04-28 20:10:58,934 INFO  [main] index.IndexCollection (IndexCollection.java:198) -  + Generator: DefaultLuceneDocumentGenerator\n",
            "2024-04-28 20:10:58,935 INFO  [main] index.IndexCollection (IndexCollection.java:199) -  + Language: pt\n",
            "2024-04-28 20:10:58,935 INFO  [main] index.IndexCollection (IndexCollection.java:200) -  + Stemmer: porter\n",
            "2024-04-28 20:10:58,935 INFO  [main] index.IndexCollection (IndexCollection.java:201) -  + Keep stopwords? false\n",
            "2024-04-28 20:10:58,936 INFO  [main] index.IndexCollection (IndexCollection.java:202) -  + Stopwords: null\n",
            "2024-04-28 20:10:58,936 INFO  [main] index.IndexCollection (IndexCollection.java:203) -  + Store positions? true\n",
            "2024-04-28 20:10:58,936 INFO  [main] index.IndexCollection (IndexCollection.java:204) -  + Store docvectors? true\n",
            "2024-04-28 20:10:58,937 INFO  [main] index.IndexCollection (IndexCollection.java:205) -  + Store document \"contents\" field? false\n",
            "2024-04-28 20:10:58,937 INFO  [main] index.IndexCollection (IndexCollection.java:206) -  + Store document \"raw\" field? true\n",
            "2024-04-28 20:10:58,938 INFO  [main] index.IndexCollection (IndexCollection.java:207) -  + Additional fields to index: []\n",
            "2024-04-28 20:10:58,938 INFO  [main] index.IndexCollection (IndexCollection.java:208) -  + Whitelist: null\n",
            "2024-04-28 20:10:58,938 INFO  [main] index.IndexCollection (IndexCollection.java:209) -  + Pretokenized?: false\n",
            "2024-04-28 20:10:58,939 INFO  [main] index.IndexCollection (IndexCollection.java:210) -  + Codec: Lucene99\n",
            "2024-04-28 20:10:58,939 INFO  [main] index.AbstractIndexer (AbstractIndexer.java:237) - ============ Indexing Collection ============\n",
            "2024-04-28 20:10:58,941 INFO  [main] index.AbstractIndexer (AbstractIndexer.java:246) - Thread pool with 9 threads initialized.\n",
            "2024-04-28 20:10:58,941 INFO  [main] index.AbstractIndexer (AbstractIndexer.java:247) - 1 file found in /content/collections\n",
            "2024-04-28 20:10:58,941 INFO  [main] index.AbstractIndexer (AbstractIndexer.java:248) - Starting to index...\n",
            "2024-04-28 20:10:59,402 DEBUG [pool-2-thread-1] index.AbstractIndexer$IndexerThread (AbstractIndexer.java:175) - collections/corpus.jsonl: 140 docs added.\n",
            "2024-04-28 20:10:59,725 INFO  [main] index.AbstractIndexer (AbstractIndexer.java:306) - Indexing Complete! 140 documents indexed\n",
            "2024-04-28 20:10:59,725 INFO  [main] index.AbstractIndexer (AbstractIndexer.java:307) - ============ Final Counter Values ============\n",
            "2024-04-28 20:10:59,725 INFO  [main] index.AbstractIndexer (AbstractIndexer.java:308) - indexed:              140\n",
            "2024-04-28 20:10:59,726 INFO  [main] index.AbstractIndexer (AbstractIndexer.java:309) - unindexable:            0\n",
            "2024-04-28 20:10:59,726 INFO  [main] index.AbstractIndexer (AbstractIndexer.java:310) - empty:                  0\n",
            "2024-04-28 20:10:59,726 INFO  [main] index.AbstractIndexer (AbstractIndexer.java:311) - skipped:                0\n",
            "2024-04-28 20:10:59,726 INFO  [main] index.AbstractIndexer (AbstractIndexer.java:312) - errors:                 0\n",
            "2024-04-28 20:10:59,732 INFO  [main] index.AbstractIndexer (AbstractIndexer.java:315) - Total 140 documents indexed in 00:00:00\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Podemos então testar a busca com BM25 para uma pergunta\n",
        "query = \"Qual é a reação da Profa. Minerva ao ouvir a confirmação de Dumbledore sobre a morte de Lílian e Tiago Potter?\"\n",
        "\n",
        "searcher = LuceneSearcher('indexes/lucene-index-trec-hp')\n",
        "hits = searcher.search(query)\n",
        "for i in range(0, 5):\n",
        "    print(f'rank:{i+1:2} doc_id: {hits[i].docid:5} score: {hits[i].score:.5f}')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zsM2EJmr9AM9",
        "outputId": "2e425bb6-c75b-49fd-82fe-5b91ce51b0f0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "rank: 1 doc_id: 7     score: 2.97870\n",
            "rank: 2 doc_id: 3     score: 2.90200\n",
            "rank: 3 doc_id: 5     score: 2.75130\n",
            "rank: 4 doc_id: 116   score: 2.73630\n",
            "rank: 5 doc_id: 131   score: 2.73460\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7JIrSy0xb-M_",
        "outputId": "16153d73-5129-417d-fd0d-a73095895ef2"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Setting BM25 parameters: k1=0.82, b=0.68\n",
            "Running topics.tsv topics, saving to runs/run.hp.bm25.lucene.txt...\n",
            "100% 140/140 [00:02<00:00, 60.12it/s]\n"
          ]
        }
      ],
      "source": [
        "# Utiliza o pacote Pyserini para ranquear todas as perguntas\n",
        "experiment = \"bm25.lucene\"\n",
        "run_file = f\"runs/run.hp.{experiment}.txt\"\n",
        "eval_file = f\"evals/eval.hp.{experiment}.txt\"\n",
        "\n",
        "!rm -rf $run_file\n",
        "!rm -rf $eval_file\n",
        "\n",
        "!python -m pyserini.search.lucene \\\n",
        "  --index \"indexes/lucene-index-trec-hp\" \\\n",
        "  --topics \"topics.tsv\" \\\n",
        "  --output $run_file \\\n",
        "  --output-format trec \\\n",
        "  --language pt \\\n",
        "  --hits 1000 \\\n",
        "  --bm25 --k1 0.82 --b 0.68\n",
        "\n",
        "# Usa o TREC_EVAL para computar as métricas\n",
        "!/content/pyserini/tools/eval/trec_eval.9.0.4/trec_eval -c -M 10 -m recip_rank -m recall.3,5,7,9 qrels.tsv $run_file > $eval_file 2>&1\n",
        "trec_aux = pd.read_csv(eval_file, sep = \"\\t\", names = [\"metric\", \"all\", \"value\"])\n",
        "trec_aux[\"experiment\"] = experiment\n",
        "trec_results[experiment] = trec_aux.copy()"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# arquivo run\n",
        "!echo $run_file\n",
        "!head -n 5 $run_file"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hQcAbtrH_4t3",
        "outputId": "1f34e41e-ed44-4456-f567-8f70282d5c51"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "runs/run.hp.bm25.lucene.txt\n",
            "0 Q0 0 1 9.661400 Anserini\n",
            "0 Q0 8 2 5.338600 Anserini\n",
            "0 Q0 2 3 5.332600 Anserini\n",
            "0 Q0 3 4 4.966200 Anserini\n",
            "0 Q0 1 5 4.086300 Anserini\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# metricas\n",
        "trec_results[experiment]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "BrcQ2eVcibAY",
        "outputId": "5261815b-9045-4ad9-f3d5-f220cf9c477c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                   metric  all   value   experiment\n",
              "0  recip_rank              all  0.8794  bm25.lucene\n",
              "1  recall_3                all  0.9143  bm25.lucene\n",
              "2  recall_5                all  0.9714  bm25.lucene\n",
              "3  recall_7                all  0.9857  bm25.lucene\n",
              "4  recall_9                all  0.9857  bm25.lucene"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-bcb2d251-9ed0-4f94-a6a0-8082f605e909\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>metric</th>\n",
              "      <th>all</th>\n",
              "      <th>value</th>\n",
              "      <th>experiment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>recip_rank</td>\n",
              "      <td>all</td>\n",
              "      <td>0.8794</td>\n",
              "      <td>bm25.lucene</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>recall_3</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9143</td>\n",
              "      <td>bm25.lucene</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>recall_5</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9714</td>\n",
              "      <td>bm25.lucene</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>recall_7</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9857</td>\n",
              "      <td>bm25.lucene</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>recall_9</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9857</td>\n",
              "      <td>bm25.lucene</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-bcb2d251-9ed0-4f94-a6a0-8082f605e909')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-bcb2d251-9ed0-4f94-a6a0-8082f605e909 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-bcb2d251-9ed0-4f94-a6a0-8082f605e909');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-78abf4f3-19a6-477e-b56b-472aba1a8102\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-78abf4f3-19a6-477e-b56b-472aba1a8102')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-78abf4f3-19a6-477e-b56b-472aba1a8102 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"trec_results[experiment]\",\n  \"rows\": 5,\n  \"fields\": [\n    {\n      \"column\": \"metric\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 5,\n        \"samples\": [\n          \"recall_3              \",\n          \"recall_9              \",\n          \"recall_5              \"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"all\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          \"all\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"value\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.048034727021187516,\n        \"min\": 0.8794,\n        \"max\": 0.9857,\n        \"num_unique_values\": 4,\n        \"samples\": [\n          0.9143\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"experiment\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          \"bm25.lucene\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dhetQiUEc2Rn"
      },
      "source": [
        "##  ADA002"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "aKMY_U2XffTF"
      },
      "outputs": [],
      "source": [
        "from google.colab import userdata\n",
        "from openai import OpenAI"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "RUkOjlazffVW"
      },
      "outputs": [],
      "source": [
        "client = OpenAI(api_key=userdata.get('openai-leonardo'))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8MP8t78odPrm"
      },
      "outputs": [],
      "source": [
        "def get_embedding(texts, model=\"text-embedding-ada-002\"):\n",
        "   texts = [text.replace(\"\\n\", \" \") for text in texts]\n",
        "   return [emb.embedding for emb in client.embeddings.create(input = texts, model=model).data]"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def ada002_search(corpus, topics):\n",
        "\n",
        "    # ada002 search\n",
        "    df = pd.DataFrame(list(corpus.items()), columns=['id', 'conteudo'])\n",
        "\n",
        "    df[\"pergunta\"] = df[\"id\"].map(topics)\n",
        "    df[\"pergunta_emb\"] = get_embedding(df[\"pergunta\"].values.tolist())\n",
        "    df[\"conteudo_emb\"] = get_embedding(df[\"conteudo\"].values.tolist())\n",
        "\n",
        "    query_matrix = np.vstack(df[\"pergunta_emb\"])\n",
        "    contents_matrix = np.vstack(df[\"conteudo_emb\"])\n",
        "    similarity_matrix = np.dot(query_matrix, contents_matrix.T)\n",
        "\n",
        "    ids = df[\"id\"].values.tolist()\n",
        "    similarity = {}\n",
        "    for query_id, hits in zip(ids, similarity_matrix):\n",
        "        if similarity.get(query_id) is None:\n",
        "            similarity[query_id] = []\n",
        "        for doc_id, hit in zip(ids, hits):\n",
        "            similarity[query_id].append((doc_id, hit))\n",
        "\n",
        "    sorted_similarity = {key: sorted(value, key=lambda x: x[1], reverse=True) for key, value in similarity.items()}\n",
        "\n",
        "    return sorted_similarity, df"
      ],
      "metadata": {
        "id": "uAhndEMiNGr2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# matriz de similaridade\n",
        "sorted_similarity, df_ada002 = ada002_search(corpus, topics)"
      ],
      "metadata": {
        "id": "9NMjqTqoBEoJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sorted_similarity[0][:5]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lRHdSiprBIVr",
        "outputId": "6de71275-65ae-4259-a3aa-9c3f149951f5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[(0, 0.8735603187132724),\n",
              " (4, 0.8618251574898236),\n",
              " (23, 0.8611035178301393),\n",
              " (103, 0.8560592778933743),\n",
              " (25, 0.8554718080588225)]"
            ]
          },
          "metadata": {},
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(f\"dimensão do embedding de cada pergunta: {len(df_ada002['pergunta_emb'][0])}\")\n",
        "print(f\"dimensão do embedding de cada chunk: {len(df_ada002['conteudo_emb'][0])}\")\n",
        "df_ada002[[\"pergunta_emb\", \"conteudo_emb\"]].head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 243
        },
        "id": "e8MOEQICBPZs",
        "outputId": "46616059-c7e5-4b7c-81b7-a46e7073902e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "dimensão do embedding de cada pergunta: 1536\n",
            "dimensão do embedding de cada chunk: 1536\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                        pergunta_emb  \\\n",
              "0  [0.022739164531230927, 0.003752939635887742, 0...   \n",
              "1  [0.0054223728366196156, 0.0008745498489588499,...   \n",
              "2  [-0.009740877896547318, -0.00151143129914999, ...   \n",
              "3  [0.003524170024320483, 0.006369490642100573, 0...   \n",
              "4  [0.0017254208214581013, 0.012094697915017605, ...   \n",
              "\n",
              "                                        conteudo_emb  \n",
              "0  [0.01479003019630909, 0.01169673353433609, -0....  \n",
              "1  [-0.0024776386562734842, -0.001506384694948792...  \n",
              "2  [0.0014590519713237882, 0.005964874755591154, ...  \n",
              "3  [0.00745804887264967, -0.0051652914844453335, ...  \n",
              "4  [0.007352123502641916, 0.003339450340718031, 0...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-daeaca58-59b8-429c-9f28-4a6143eff8fd\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>pergunta_emb</th>\n",
              "      <th>conteudo_emb</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>[0.022739164531230927, 0.003752939635887742, 0...</td>\n",
              "      <td>[0.01479003019630909, 0.01169673353433609, -0....</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>[0.0054223728366196156, 0.0008745498489588499,...</td>\n",
              "      <td>[-0.0024776386562734842, -0.001506384694948792...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>[-0.009740877896547318, -0.00151143129914999, ...</td>\n",
              "      <td>[0.0014590519713237882, 0.005964874755591154, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>[0.003524170024320483, 0.006369490642100573, 0...</td>\n",
              "      <td>[0.00745804887264967, -0.0051652914844453335, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>[0.0017254208214581013, 0.012094697915017605, ...</td>\n",
              "      <td>[0.007352123502641916, 0.003339450340718031, 0...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-daeaca58-59b8-429c-9f28-4a6143eff8fd')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-daeaca58-59b8-429c-9f28-4a6143eff8fd button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-daeaca58-59b8-429c-9f28-4a6143eff8fd');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-6f4bdfcc-da36-4eb8-9f9e-b8da8275858f\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-6f4bdfcc-da36-4eb8-9f9e-b8da8275858f')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-6f4bdfcc-da36-4eb8-9f9e-b8da8275858f button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"df_ada002[[\\\"pergunta_emb\\\", \\\"conteudo_emb\\\"]]\",\n  \"rows\": 5,\n  \"fields\": [\n    {\n      \"column\": \"pergunta_emb\",\n      \"properties\": {\n        \"dtype\": \"object\",\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"conteudo_emb\",\n      \"properties\": {\n        \"dtype\": \"object\",\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Save run file\n",
        "experiment = \"ada002\"\n",
        "run_file = f\"runs/run.hp.{experiment}.txt\"\n",
        "eval_file = f\"evals/eval.hp.{experiment}.txt\"\n",
        "\n",
        "!rm -rf $run_file\n",
        "!rm -rf $eval_file\n",
        "\n",
        "with open(run_file,'w') as f_out:\n",
        "    for query_id, hits in tqdm(sorted_similarity.items(), desc=f'Writing file on {run_file}'):\n",
        "        rank = 1\n",
        "        for doc_id, score in hits:\n",
        "            f_out.write(f'{query_id}\\tQ0\\t{doc_id}\\t{rank}\\t{score}\\t{\"ADA002\"}\\n')\n",
        "            rank+=1\n",
        "\n",
        "# Usa o TREC_EVAL para computar as métricas\n",
        "!/content/pyserini/tools/eval/trec_eval.9.0.4/trec_eval -c -M 10 -m recip_rank -m recall.3,5,7,9 qrels.tsv $run_file > $eval_file 2>&1\n",
        "trec_aux = pd.read_csv(eval_file, sep = \"\\t\", names = [\"metric\", \"all\", \"value\"])\n",
        "trec_aux[\"experiment\"] = experiment\n",
        "trec_results[experiment] = trec_aux.copy()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4JeheNTaotBL",
        "outputId": "1a242e18-0c9c-43bb-a435-c287a5e6fc40"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Writing file on runs/run.hp.ada002.txt: 100%|██████████| 140/140 [00:00<00:00, 4164.68it/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# arquivo run\n",
        "!head -n 5 $run_file"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yO4rKILlCAhl",
        "outputId": "4892e93b-40d1-4906-b014-70c4ae8a07d8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0\tQ0\t0\t1\t0.8735603187132724\tADA002\n",
            "0\tQ0\t4\t2\t0.8618251574898236\tADA002\n",
            "0\tQ0\t23\t3\t0.8611035178301393\tADA002\n",
            "0\tQ0\t103\t4\t0.8560592778933743\tADA002\n",
            "0\tQ0\t25\t5\t0.8554718080588225\tADA002\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "trec_results[experiment]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "PUQfkJWLU66B",
        "outputId": "b6e7884d-5271-41f1-b101-5900be351776"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                   metric  all   value experiment\n",
              "0  recip_rank              all  0.5645     ada002\n",
              "1  recall_3                all  0.6214     ada002\n",
              "2  recall_5                all  0.6929     ada002\n",
              "3  recall_7                all  0.7500     ada002\n",
              "4  recall_9                all  0.8071     ada002"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-d652edfd-7338-4816-9e89-437b89621bab\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>metric</th>\n",
              "      <th>all</th>\n",
              "      <th>value</th>\n",
              "      <th>experiment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>recip_rank</td>\n",
              "      <td>all</td>\n",
              "      <td>0.5645</td>\n",
              "      <td>ada002</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>recall_3</td>\n",
              "      <td>all</td>\n",
              "      <td>0.6214</td>\n",
              "      <td>ada002</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>recall_5</td>\n",
              "      <td>all</td>\n",
              "      <td>0.6929</td>\n",
              "      <td>ada002</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>recall_7</td>\n",
              "      <td>all</td>\n",
              "      <td>0.7500</td>\n",
              "      <td>ada002</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>recall_9</td>\n",
              "      <td>all</td>\n",
              "      <td>0.8071</td>\n",
              "      <td>ada002</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-d652edfd-7338-4816-9e89-437b89621bab')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-d652edfd-7338-4816-9e89-437b89621bab button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-d652edfd-7338-4816-9e89-437b89621bab');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-9731fee9-93d7-4900-af3e-f5f3e4d77e82\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-9731fee9-93d7-4900-af3e-f5f3e4d77e82')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-9731fee9-93d7-4900-af3e-f5f3e4d77e82 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"trec_results[experiment]\",\n  \"rows\": 5,\n  \"fields\": [\n    {\n      \"column\": \"metric\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 5,\n        \"samples\": [\n          \"recall_3              \",\n          \"recall_9              \",\n          \"recall_5              \"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"all\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          \"all\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"value\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.09713041233311019,\n        \"min\": 0.5645,\n        \"max\": 0.8071,\n        \"num_unique_values\": 5,\n        \"samples\": [\n          0.6214\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"experiment\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          \"ada002\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "trec_results[experiment]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "v0aKlJq7LNLR",
        "outputId": "b81a765d-0d6a-4b28-d583-b86570cbe5c2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                   metric  all   value experiment\n",
              "0  recip_rank              all  0.5645     ada002\n",
              "1  recall_3                all  0.6214     ada002\n",
              "2  recall_5                all  0.6929     ada002\n",
              "3  recall_7                all  0.7500     ada002\n",
              "4  recall_9                all  0.8071     ada002"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-d9561c13-d371-4d36-b304-35e26ae14f5a\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>metric</th>\n",
              "      <th>all</th>\n",
              "      <th>value</th>\n",
              "      <th>experiment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>recip_rank</td>\n",
              "      <td>all</td>\n",
              "      <td>0.5645</td>\n",
              "      <td>ada002</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>recall_3</td>\n",
              "      <td>all</td>\n",
              "      <td>0.6214</td>\n",
              "      <td>ada002</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>recall_5</td>\n",
              "      <td>all</td>\n",
              "      <td>0.6929</td>\n",
              "      <td>ada002</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>recall_7</td>\n",
              "      <td>all</td>\n",
              "      <td>0.7500</td>\n",
              "      <td>ada002</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>recall_9</td>\n",
              "      <td>all</td>\n",
              "      <td>0.8071</td>\n",
              "      <td>ada002</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-d9561c13-d371-4d36-b304-35e26ae14f5a')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-d9561c13-d371-4d36-b304-35e26ae14f5a button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-d9561c13-d371-4d36-b304-35e26ae14f5a');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-3269f7ba-0ee8-40d2-9b6e-71442663b3f5\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-3269f7ba-0ee8-40d2-9b6e-71442663b3f5')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-3269f7ba-0ee8-40d2-9b6e-71442663b3f5 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"trec_results[experiment]\",\n  \"rows\": 5,\n  \"fields\": [\n    {\n      \"column\": \"metric\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 5,\n        \"samples\": [\n          \"recall_3              \",\n          \"recall_9              \",\n          \"recall_5              \"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"all\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          \"all\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"value\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.09713041233311019,\n        \"min\": 0.5645,\n        \"max\": 0.8071,\n        \"num_unique_values\": 5,\n        \"samples\": [\n          0.6214\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"experiment\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          \"ada002\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Hybrid Search (BM25 + ADA002)\n",
        "\n",
        "* https://plg.uwaterloo.ca/~gvcormac/cormacksigir09-rrf.pdf"
      ],
      "metadata": {
        "id": "0BT0FQykqfBO"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# save run file\n",
        "rrf_k_list = [60]\n",
        "\n",
        "for rrf_k in rrf_k_list:\n",
        "    experiment = f\"hybrid_bm25_ada002_k_{rrf_k}\"\n",
        "    run_file = f'runs/run.hp.{experiment}.txt'\n",
        "    eval_file = f'evals/eval.hp.{experiment}.txt'\n",
        "\n",
        "    run_file1_path = 'runs/run.hp.bm25.lucene.txt'\n",
        "    run_file2_path = 'runs/run.hp.ada002.txt'\n",
        "\n",
        "    !python -m pyserini.fusion \\\n",
        "      --runs $run_file1_path $run_file2_path \\\n",
        "      --output $run_file \\\n",
        "      --rrf.k $rrf_k \\\n",
        "      --k 1000\n",
        "\n",
        "    # Usa o TREC_EVAL para computar as métricas\n",
        "    !/content/pyserini/tools/eval/trec_eval.9.0.4/trec_eval -c -M 10 -m recip_rank -m recall.3,5,7,9 qrels.tsv $run_file > $eval_file 2>&1\n",
        "    trec_aux = pd.read_csv(eval_file, sep = \"\\t\", names = [\"metric\", \"all\", \"value\"])\n",
        "    trec_aux[\"experiment\"] = experiment\n",
        "    trec_results[experiment] = trec_aux.copy()"
      ],
      "metadata": {
        "id": "umzUkommzLb-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "rrf_k_list = [60]\n",
        "for rrf_k in rrf_k_list:\n",
        "    experiment = f\"hybrid_bm25_ada002_k_{rrf_k}\"\n",
        "    display(trec_results[experiment])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "ah698XsVIqF5",
        "outputId": "af6bf8a3-262d-4415-f34a-ea6ec18a1360"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "                   metric  all   value               experiment\n",
              "0  recip_rank              all  0.7587  hybrid_bm25_ada002_k_60\n",
              "1  recall_3                all  0.8357  hybrid_bm25_ada002_k_60\n",
              "2  recall_5                all  0.8786  hybrid_bm25_ada002_k_60\n",
              "3  recall_7                all  0.9143  hybrid_bm25_ada002_k_60\n",
              "4  recall_9                all  0.9571  hybrid_bm25_ada002_k_60"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-12288869-c1d7-477a-a5e0-c2109c28b415\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>metric</th>\n",
              "      <th>all</th>\n",
              "      <th>value</th>\n",
              "      <th>experiment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>recip_rank</td>\n",
              "      <td>all</td>\n",
              "      <td>0.7587</td>\n",
              "      <td>hybrid_bm25_ada002_k_60</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>recall_3</td>\n",
              "      <td>all</td>\n",
              "      <td>0.8357</td>\n",
              "      <td>hybrid_bm25_ada002_k_60</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>recall_5</td>\n",
              "      <td>all</td>\n",
              "      <td>0.8786</td>\n",
              "      <td>hybrid_bm25_ada002_k_60</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>recall_7</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9143</td>\n",
              "      <td>hybrid_bm25_ada002_k_60</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>recall_9</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9571</td>\n",
              "      <td>hybrid_bm25_ada002_k_60</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-12288869-c1d7-477a-a5e0-c2109c28b415')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-12288869-c1d7-477a-a5e0-c2109c28b415 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-12288869-c1d7-477a-a5e0-c2109c28b415');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-e9b970a2-9119-4d07-bc5b-b46f76492f51\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-e9b970a2-9119-4d07-bc5b-b46f76492f51')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-e9b970a2-9119-4d07-bc5b-b46f76492f51 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"    display(trec_results[experiment])\",\n  \"rows\": 5,\n  \"fields\": [\n    {\n      \"column\": \"metric\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 5,\n        \"samples\": [\n          \"recall_3              \",\n          \"recall_9              \",\n          \"recall_5              \"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"all\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          \"all\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"value\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.07612615844767157,\n        \"min\": 0.7587,\n        \"max\": 0.9571,\n        \"num_unique_values\": 5,\n        \"samples\": [\n          0.8357\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"experiment\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          \"hybrid_bm25_ada002_k_60\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## ADA002 CUSTOM"
      ],
      "metadata": {
        "id": "LVFrlYg__e4G"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def embedding_multiplied_by_matrix(embedding, matrix):\n",
        "    embedding_tensor = torch.tensor(embedding).float()\n",
        "    modified_embedding = torch.einsum(\"b, bc -> c\", embedding_tensor, matrix)\n",
        "\n",
        "    modified_embedding = modified_embedding.detach().numpy()\n",
        "\n",
        "    return modified_embedding\n",
        "\n",
        "\n",
        "def apply_matrix_to_embeddings_dataframe(matrix, df):\n",
        "    def cosine_similarity(a, b):\n",
        "        return np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b))\n",
        "\n",
        "\n",
        "    for column in [\"pergunta_embedding\", \"chunk_embedding\"]:\n",
        "        df[f\"{column}_custom\"] = df[column].apply(\n",
        "            lambda x: embedding_multiplied_by_matrix(x, matrix)\n",
        "        )\n",
        "    df[\"cosine_similarity_custom\"] = df.apply(\n",
        "        lambda row: cosine_similarity(\n",
        "            row[\"pergunta_embedding_custom\"], row[\"chunk_embedding_custom\"]\n",
        "        ),\n",
        "        axis=1,\n",
        "    )"
      ],
      "metadata": {
        "id": "kinQh85RB1Fv"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df_custom = pd.read_csv(f\"{path}/tozip/df_final_to_customize_embds_HP_2negs.csv\")"
      ],
      "metadata": {
        "id": "eJOKV_Qb_now"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df_custom.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 293
        },
        "id": "iHNcQdekCKwR",
        "outputId": "14478034-8b44-4df3-a36a-4ded7100291f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                            pergunta  \\\n",
              "0  Qual é o segredo que o Sr. e a Sra. Dursley te...   \n",
              "1  Qual era a cor da capa que o Sr. Dursley viu u...   \n",
              "2  Qual foi a reação do Sr. Dursley ao ser abraça...   \n",
              "3  Qual é o nome completo do personagem que apare...   \n",
              "4  Qual objeto Dumbledore utilizou para apagar os...   \n",
              "\n",
              "                                            contexto  \\\n",
              "0  Os Dursley tinham tudo que queriam, mas tinham...   \n",
              "1  O Sr. Dursley não tolerava gente que andava co...   \n",
              "2  Levou alguns segundos até o Sr. Dursley perceb...   \n",
              "3  Ninguém jamais vislumbrara nada parecido com e...   \n",
              "4  Encontrou o que procurava no bolso interior da...   \n",
              "\n",
              "                                            resposta  \\\n",
              "0                            a existência dos Potter   \n",
              "1                           uma capa verde-esmeralda   \n",
              "2  O Sr. Dursley ficou pregado no chão. Fora abra...   \n",
              "3                   O nome dele era Alvo Dumbledore.   \n",
              "4              Ele acionou o 'apagueiro' doze vezes.   \n",
              "\n",
              "                                               chunk  label dataset  \\\n",
              "0  - CAPÍTULO UM - O menino que sobreviveu O Sr. ...      1   train   \n",
              "1  ante o caminho para a cidade ele não pensou em...      1   train   \n",
              "2  em alguém parado ali à porta. - Desculpe - mur...      1   train   \n",
              "3  orrecida. Afinal, normalmente fingiam que ela ...      1   train   \n",
              "4  pando a capa, procurando alguma coisa. Mas par...      1   train   \n",
              "\n",
              "                                  pergunta_embedding  \\\n",
              "0  [0.022689364850521088, 0.0038326631765812635, ...   \n",
              "1  [0.005216388031840324, 0.0008732206770218909, ...   \n",
              "2  [-0.009695776738226414, -0.0014956077793613076...   \n",
              "3  [0.004215216264128685, 0.00600033113732934, 0....   \n",
              "4  [0.001769782043993473, 0.012089592404663563, 0...   \n",
              "\n",
              "                                     chunk_embedding  cosine_similarity  \n",
              "0  [0.014419169165194035, 0.011491919867694378, -...           0.873794  \n",
              "1  [-0.002306348877027631, -0.0014758666511625051...           0.873999  \n",
              "2  [0.004920588340610266, 0.004709234461188316, 0...           0.884103  \n",
              "3  [0.008049095049500465, -0.0045163994655013084,...           0.812161  \n",
              "4  [0.007251789793372154, 0.003910213243216276, 0...           0.879149  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-42e1441c-7179-4e37-918c-67ca984ad615\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>pergunta</th>\n",
              "      <th>contexto</th>\n",
              "      <th>resposta</th>\n",
              "      <th>chunk</th>\n",
              "      <th>label</th>\n",
              "      <th>dataset</th>\n",
              "      <th>pergunta_embedding</th>\n",
              "      <th>chunk_embedding</th>\n",
              "      <th>cosine_similarity</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Qual é o segredo que o Sr. e a Sra. Dursley te...</td>\n",
              "      <td>Os Dursley tinham tudo que queriam, mas tinham...</td>\n",
              "      <td>a existência dos Potter</td>\n",
              "      <td>- CAPÍTULO UM - O menino que sobreviveu O Sr. ...</td>\n",
              "      <td>1</td>\n",
              "      <td>train</td>\n",
              "      <td>[0.022689364850521088, 0.0038326631765812635, ...</td>\n",
              "      <td>[0.014419169165194035, 0.011491919867694378, -...</td>\n",
              "      <td>0.873794</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Qual era a cor da capa que o Sr. Dursley viu u...</td>\n",
              "      <td>O Sr. Dursley não tolerava gente que andava co...</td>\n",
              "      <td>uma capa verde-esmeralda</td>\n",
              "      <td>ante o caminho para a cidade ele não pensou em...</td>\n",
              "      <td>1</td>\n",
              "      <td>train</td>\n",
              "      <td>[0.005216388031840324, 0.0008732206770218909, ...</td>\n",
              "      <td>[-0.002306348877027631, -0.0014758666511625051...</td>\n",
              "      <td>0.873999</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Qual foi a reação do Sr. Dursley ao ser abraça...</td>\n",
              "      <td>Levou alguns segundos até o Sr. Dursley perceb...</td>\n",
              "      <td>O Sr. Dursley ficou pregado no chão. Fora abra...</td>\n",
              "      <td>em alguém parado ali à porta. - Desculpe - mur...</td>\n",
              "      <td>1</td>\n",
              "      <td>train</td>\n",
              "      <td>[-0.009695776738226414, -0.0014956077793613076...</td>\n",
              "      <td>[0.004920588340610266, 0.004709234461188316, 0...</td>\n",
              "      <td>0.884103</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Qual é o nome completo do personagem que apare...</td>\n",
              "      <td>Ninguém jamais vislumbrara nada parecido com e...</td>\n",
              "      <td>O nome dele era Alvo Dumbledore.</td>\n",
              "      <td>orrecida. Afinal, normalmente fingiam que ela ...</td>\n",
              "      <td>1</td>\n",
              "      <td>train</td>\n",
              "      <td>[0.004215216264128685, 0.00600033113732934, 0....</td>\n",
              "      <td>[0.008049095049500465, -0.0045163994655013084,...</td>\n",
              "      <td>0.812161</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Qual objeto Dumbledore utilizou para apagar os...</td>\n",
              "      <td>Encontrou o que procurava no bolso interior da...</td>\n",
              "      <td>Ele acionou o 'apagueiro' doze vezes.</td>\n",
              "      <td>pando a capa, procurando alguma coisa. Mas par...</td>\n",
              "      <td>1</td>\n",
              "      <td>train</td>\n",
              "      <td>[0.001769782043993473, 0.012089592404663563, 0...</td>\n",
              "      <td>[0.007251789793372154, 0.003910213243216276, 0...</td>\n",
              "      <td>0.879149</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-42e1441c-7179-4e37-918c-67ca984ad615')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-42e1441c-7179-4e37-918c-67ca984ad615 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-42e1441c-7179-4e37-918c-67ca984ad615');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-5f75527d-94ce-4244-8177-88e24f8d0b1c\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-5f75527d-94ce-4244-8177-88e24f8d0b1c')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-5f75527d-94ce-4244-8177-88e24f8d0b1c button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "df_custom",
              "summary": "{\n  \"name\": \"df_custom\",\n  \"rows\": 420,\n  \"fields\": [\n    {\n      \"column\": \"pergunta\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 202,\n        \"samples\": [\n          \"amente evitara encarar Harry. Resmungou alguma coisa e lhe ofereceu mais um biscoito. Harry releu a not\\u00edcia. O cofre aberto na realidade fora esvaziado mais cedo naquele dia. Hagrid esvaziara o cofre setecentos e treze, se \\u00e9 que se podia chamar esvaziar algu\\u00e9m levar aquele pacotinho encalombado. Seria aquilo que os ladr\\u00f5es estavam procurando? Quando Harry e Rony voltaram ao castelo para jantar, tinham os bolsos pesados com os biscoitos que a educa\\u00e7\\u00e3o os impedira de recusar. Harry pensou que nenhuma das aulas a que assistira at\\u00e9 ali tinha-lhe dado tanto o que pensar quanto o ch\\u00e1 com R\\u00fabeo Hagrid. Ser\\u00e1 que Hagrid tinha apanhado o pacote bem na hora? Onde estava o pacote agora? Ser\\u00e1 que ele sabia alguma coisa de Snape que n\\u00e3o queria contar a Harry? - CAP\\u00cdTULO NOVE - O duelo \\u00e0 meia-noite Harry jamais acreditara que fosse encontrar um garoto que ele detestasse mais do que Duda, mas isto foi antes de conhecer Draco. Os alunos do primeiro ano da Grifin\\u00f3ria, por\\u00e9m, s\\u00f3 tinham uma aula com os da Sonserina, a de Po\\u00e7\\u00f5es, por isso n\\u00e3o precisavam aturar Draco muito tempo. Ou pelo menos, n\\u00e3o precisavam at\\u00e9 verem um aviso pregado no sala comunal de Grifin\\u00f3ria que fez todos gemerem. As aulas de voo come\\u00e7ariam na quinta-feira - e os alunos das duas casas aprenderiam juntos. - T\\u00edpico - disse Harry, desanimado. - \\u00c9 o que eu sempre quis. Fazer papel de palha\\u00e7o montado numa vassoura na frente do Draco. Ele estivera ansioso para aprender a voar, mais do que qualquer outra coisa. - Voc\\u00ea n\\u00e3o sabe se vai fazer papel de palha\\u00e7o - disse Rony, sensato. - Em todo o caso, sei que Draco vive falando que \\u00e9 bom em quadribol, mas aposto que \\u00e9 conversa fiada. Draco sem d\\u00favida falava muito de voos. Queixava-se em voz alta que os alunos do primeiro ano nunca entravam para o time de quadribol e se gabava em longas hist\\u00f3rias, que sempre pareciam terminar com ele escapando por um triz dos trouxas de helic\\u00f3pteros. Mas ele n\\u00e3o era o \\u00fanico: pelo que Simas Finnigan contava, ele passara a maior parte da inf\\u00e2ncia voando pelo campo montado numa vassoura. At\\u00e9 Rony contava para quem quisesse ouvir sobre a vez em que ele quase batera numa asa delta montado na velha vassoura de Carlinhos. Todos os garotos de fam\\u00edlias de bruxos falavam o tempo todo de quadribol. Rony j\\u00e1 tivera uma grande discuss\\u00e3o sobre futebol com Dino Thomas, que tamb\\u00e9m usava o dormit\\u00f3rio deles. Rony n\\u00e3o via nada excitante em um jogo em que ningu\\u00e9m podia voar e s\\u00f3 tinha uma bola. Harry surpreendera Rony cutucando o p\\u00f4ster em que Dino aparecia com o time de futebol de West Ham, tentando fazer os jogadores se mexerem. Neville nunca andara de vassoura na vida, porque a av\\u00f3 nunca o deixara chegar perto de uma. No fundo, Harry achava que ela estava cert\\u00edssima, porque Neville conseguira sofrer um n\\u00famero impressionante de acidentes mesmo com os dois p\\u00e9s no ch\\u00e3o. Hermione Granger estava quase t\\u00e3o nervosa quanto Neville com a ideia de voar. Isto n\\u00e3o era coisa que se aprendesse de cor em um livro - n\\u00e3o que ela n\\u00e3o tivesse tentado. No caf\\u00e9 da manh\\u00e3 de quinta-feira, deu um cansa\\u00e7o neles falando sobre macetes de voo que lera em um livro da biblioteca chamado Quadribol atrav\\u00e9s dos s\\u00e9culos. Neville praticamente se pendurava em cada palavra que ela dizia, desesperado para aprender qualquer coisa que o ajudasse a se segurar na vassoura mais tarde, mas todos os outros ficaram muito felizes quando a confer\\u00eancia de Hermione foi interrompida pela chegada do correio. Harry n\\u00e3o recebera nenhuma carta desde o bilh\",\n          \"Qual \\u00e9 o endere\\u00e7o completo de Harry Potter conforme indicado na carta que ele recebeu, incluindo o nome do destinat\\u00e1rio, a localiza\\u00e7\\u00e3o espec\\u00edfica da casa e o c\\u00f3digo postal?\",\n          \"Qual \\u00e9 o nome do bar que Hagrid descreve como um lugar famoso, mas que Harry percebe ser muito escuro e miser\\u00e1vel quando o visitam pela primeira vez?\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"contexto\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 140,\n        \"samples\": [\n          \"A Profa. Minerva, num robe de l\\u00e3 escocesa e rede no cabelo, segurava Malfoy pela orelha. - Est\\u00e1 detido - gritou. - E s\\u00e3o vinte pontos a menos para Sonserina. Perambulando no meio da noite, como voc\\u00ea se atreve...\",\n          \"Ele perdeu a anima\\u00e7\\u00e3o mais depressa do que quando mergulhara. A Profa. Minerva vinha correndo em dire\\u00e7\\u00e3o \\u00e0 turma. Ele se levantou tremendo. - Nunca... em todo o tempo que estou em Hogwarts... A Profa. Minerva quase perdeu a fala de espanto e seus \\u00f3culos cintilavam sem parar, \\\"... como \\u00e9 que voc\\u00ea se atreve... podia ter partido o pesco\\u00e7o...\\\" .\",\n          \"Um rapaz p\\u00e1lido adiantou-se, muito nervoso. Um olho tr\\u00eamulo. - Prof. Quirrell! - disse Hagrid. - Harry, o Prof. Quirrell vai ser um dos seus professores em Hogwarts. - P-P-Potter - gaguejou o Prof. Quirrell, apertando a m\\u00e3o de Harry -, n-n-em sei d-d-dizer que p-p-p-prazer enorme \\u00e9 c-c-conhec\\u00ea-lo. - Que tipo de m\\u00e1gica o senhor ensina, Prof. Quirrell? - D-d-defesa C-c-ontra as Art-t-tes das T-t-trevas - murmurou o Prof. Quirrell, como se preferisse n\\u00e3o pensar no assunto.\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"resposta\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 140,\n        \"samples\": [\n          \"Est\\u00e1 detido - gritou. - E s\\u00e3o vinte pontos a menos para Sonserina.\",\n          \"A Profa. Minerva quase perdeu a fala de espanto e seus \\u00f3culos cintilavam sem parar, \\\"... como \\u00e9 que voc\\u00ea se atreve... podia ter partido o pesco\\u00e7o...\\\"\",\n          \"Prof. Quirrell ensina Defesa Contra as Artes das Trevas.\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"chunk\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 210,\n        \"samples\": [\n          \": Um guia de autoprote\\u00e7\\u00e3o de Quintino Trimble. Outros Equipamentos 1 varinha m\\u00e1gica 1 caldeir\\u00e3o (estanho, tamanho padr\\u00e3o 2) 1 conjunto de frascos 1 telesc\\u00f3pio 1 balan\\u00e7a de lat\\u00e3o Os alunos podem ainda trazer uma coruja OU um gato OU um sapo. LEMBRAMOS AOS PAIS QUE OS ALUNOS DO PRIMEIRO ANO N\\u00c3O PODEM USAR VASSOURAS PESSOAIS. - Podemos comprar tudo isso em Londres? - perguntou-se Harry em voz alta. - Se voc\\u00ea souber aonde ir - respondeu Hagrid. Harry nunca estivera em Londres antes. Hagrid, embora parecesse saber aonde ia, obviamente n\\u00e3o estava acostumado a chegar l\\u00e1 pelos meios comuns. Ficou entalado na roleta do metr\\u00f4 e queixou-se em voz alta que os assentos eram demasiado pequenos e os trens demasiado lentos. - N\\u00e3o sei como os trouxas conseguem se arranjar sem m\\u00e1gica - disse, quando subiam uma escada rolante gasta que levava a uma rua movimentada com lojas dos dois lados. Hagrid era t\\u00e3o grande que abria caminho pela multid\\u00e3o sem esfor\\u00e7o, Harry s\\u00f3 precisava segui-lo de perto. Passaram por livrarias e lojas de m\\u00fasica, lanchonetes e cinemas, mas nenhuma loja parecia vender varinhas m\\u00e1gicas. Aquela era apenas uma rua comum cheia de gente comum. Seria realmente poss\\u00edvel que houvesse montes de ouro dos bruxos enterrados quil\\u00f4metros abaixo dali? Haveria realmente lojas que vendessem livros de feiti\\u00e7os e vassouras? N\\u00e3o seria talvez uma grande pe\\u00e7a que os Dursley tinham pregado? Se Harry n\\u00e3o soubesse que os Dursley n\\u00e3o tinham senso de humor, poderia ter tirado uma dessas conclus\\u00f5es; mas, por alguma raz\\u00e3o, embora tudo que Hagrid tivesse dito at\\u00e9 ali fosse inacredit\\u00e1vel, Harry n\\u00e3o podia deixar de confiar nele. - \\u00c9 aqui - disse Hagrid parando. - O Caldeir\\u00e3o Furado. \\u00c9 um lugar famoso. Era um barzinho sujo. Se Hagrid n\\u00e3o o tivesse apontado, Harry nem teria reparado que existia. As pessoas que passavam apressadas nem olhavam para aquele lado. Os olhos delas corriam da grande livraria a um lado \\u00e0 loja de discos no outro como se nem conseguissem ver O Caldeir\\u00e3o Furado. Na verdade Harry teve a sensa\\u00e7\\u00e3o muito estranha de que somente ele e Hagrid eram capazes de v\\u00ea- lo. Antes que pudesse comentar isto, Hagrid o empurrou para dentro. Para um lugar famoso, o Caldeir\\u00e3o era muito escuro e miser\\u00e1vel. Havia umas velhas sentadas a um canto, bebendo pequenos c\\u00e1lices de xerez. Uma delas fumava um longo cachimbo. Um homenzinho de cartola conversava com o velho dono do bar, que era bem careca e parecia uma noz viscosa. O zum-zum das conversas parou quando eles entraram. Todos pareciam conhecer Hagrid; acenaram e sorriram para ele, Tom apanhou um copo, perguntando: - O de sempre, Hagrid? - N\\u00e3o posso, Tom, estou a servi\\u00e7o de Hogwarts - disse Hagrid, dando uma palmada com a manzorra no ombro de Harry, o que fez os joelhos do garoto dobrarem. - Meu Deus! - exclamou Tom, fitando Harry. - \\u00c9... ser\\u00e1 poss\\u00edvel? O Caldeir\\u00e3o Furado repentinamente parou e fez-se um sil\\u00eancio total. - Valha-me Deus - murmurou o velho Tom. - Harry Potter... que honra. E saiu correndo de tr\\u00e1s do balc\\u00e3o, precipitou-se para Harry e agarrou suas m\\u00e3os, as l\\u00e1grimas nos olhos. - Seja bem-vindo, Sr. Potter, seja bem-vindo. Harry n\\u00e3o sabia o que dizer. Todos tinham os olhos nele. A velha com o cachimbo puxava o f\",\n          \"a ele n\\u00e3o tinham mudado nem um pingo. Isto n\\u00e3o o preocupou. Parecia-lhe que sua vida voltaria ao normal no pr\\u00f3ximo ano, ou t\\u00e3o normal quanto ela poderia ser em Hogwarts. Foi a melhor noite da vida de Harry, melhor do que a vit\\u00f3ria no quadribol ou a ceia de Natal ou o encontro com o trasgo... jamais esqueceria esta noite. Harry quase esquecera que os resultados dos exames ainda estavam por vir, mas eles n\\u00e3o deixaram de vir. Para sua grande surpresa, tanto ele quanto Rony passaram com boas notas; Hermione, \\u00e9 claro, foi a melhor do ano. At\\u00e9 Neville passou raspando, sua boa nota em Herbologia compensou a p\\u00e9ssima nota em Po\\u00e7\\u00f5es. Tinham tido esperan\\u00e7as de que Goyle, que era quase t\\u00e3o burro quanto era mau, fosse expulso, mas ele tamb\\u00e9m passou. Foi uma pena, mas, como disse Rony, n\\u00e3o se podia ter tudo na vida. E, de repente, seus guarda-roupas ficaram vazios, os mal\\u00f5es arrumados, o sapo de Neville foi encontrado escondido em um canto do banheiro; as notas foram entregues a todos os alunos, com o aviso de que n\\u00e3o fizessem bruxarias durante as f\\u00e9rias (\\\"Eu sempre tenho a esperan\\u00e7a de que eles se esque\\u00e7am de entregar as notas e o aviso\\\" - lamentou Fred Weasley); Hagrid estava a postos para lev\\u00e1-los \\u00e0 flotilha de barcos que fazia a travessia do lago; e, no momento seguinte, estavam embarcando no Expresso de Hogwarts; conversando e rindo \\u00e0 medida que os campos se tornavam mais verdes e mais cuidados; comendo feij\\u00f5ezinhos de todos os sabores enquanto atravessavam as cidades dos trouxas; trocando as vestes de bruxos pelos blus\\u00f5es e palet\\u00f3s; parando na plataforma nove e meia na esta\\u00e7\\u00e3o de King\\\"s Cross. Levou um bom tempo para todos desembarcarem na plataforma. Um guarda muito velho estava postado na sa\\u00edda e os deixava passar em grupos de dois e tr\\u00eas para n\\u00e3o chamarem aten\\u00e7\\u00e3o ao irromper todos ao mesmo tempo por uma parede s\\u00f3lida, assustando os trouxas. - Voc\\u00eas precisam vir passar uns dias conosco - disse Rony. - Os dois. Vou mandar uma coruja para voc\\u00eas. - Obrigado - disse Harry. - Preciso ter alguma coisa por que esperar. As pessoas passavam empurrando-os ao se dirigirem para a sa\\u00edda para o mundo dos trouxas. Alguns gritavam: - Tchau, Harry! - Nos vemos por a\\u00ed, Potter! - Continua famoso - comentou Rony, sorrindo para o amigo. - N\\u00e3o aonde eu vou, posso lhe garantir. Ele, Rony e Hermione passaram juntos pelo port\\u00e3o. - Olha l\\u00e1 ele, mam\\u00e3e, olha l\\u00e1 ele, olha! Era Gina Weasley, a irm\\u00e3zinha de Rony, mas n\\u00e3o apontava para Rony. - Harry Potter! - gritou com a vozinha fina. - Olhe, mam\\u00e3e! Estou vendo... - Fique quieta, Gina, \\u00e9 falta de educa\\u00e7\\u00e3o apontar. A Sra. Weasley sorriu para eles. - Muito trabalho este ano? - Muito - respondeu Harry. - Obrigado pelas barrinhas de chocolate e pela su\\u00e9ter, Sra. Weasley. - Ah, de nada, querido. - Est\\u00e1 pronto? Era tio V\\u00e1lter, ainda com a cara vermelhona, ainda bigodudo, ainda parecendo furioso com a aud\\u00e1cia de Harry de andar carregando uma coruja numa gaiola por uma esta\\u00e7\\u00e3o cheia de gente normal. Atr\\u00e1s dele, achavam-se tia Pet\\u00fania e Duda, parecendo aterrorizados s\\u00f3 de olhar para Harry. - Voc\\u00eas devem ser a fam\\u00edlia de Harry! - falou a Sra. Weasley. - Por assim dizer - respondeu tio V\\u00e1lter. - Ande logo, menino, n\\u00e3o temos o dia inteiro. - E se afastou. Harry ainda demorou para trocar uma \\u00faltima palavrinha com Rony e Hermione.\",\n          \"Qual foi a rea\\u00e7\\u00e3o de tio V\\u00e1lter quando Harry mencionou ter sonhado com uma motocicleta que voava?\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"label\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0,\n        \"min\": -1,\n        \"max\": 1,\n        \"num_unique_values\": 2,\n        \"samples\": [\n          -1,\n          1\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"dataset\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 2,\n        \"samples\": [\n          \"test\",\n          \"train\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"pergunta_embedding\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 324,\n        \"samples\": [\n          \"[-0.010704532265663147, -0.02066177874803543, 0.005860289093106985, -0.018931224942207336, 0.0020910874009132385, 0.019730951637029648, -0.009301734156906605, -0.026731831952929497, -0.023388715460896492, 0.0032841216307133436, 0.015837201848626137, -0.018315041437745094, -0.015666767954826355, -0.013103711418807507, 0.015011254698038101, 0.007971041835844517, 0.01277595479041338, 0.005060562863945961, 0.013352805748581886, -0.0203864648938179, -0.008600334636867046, 0.013582236133515835, 0.007223756518214941, -0.010933961719274521, -0.0011840208899229765, -0.010953627526760101, 0.002720380201935768, -0.020950205624103546, 0.028973689302802086, -0.019376972690224648, 0.015942083671689034, 0.004074015188962221, -0.013818220235407352, 0.0006088080117478967, -0.031674403697252274, -0.004496821202337742, 0.009105079807341099, 0.01564054749906063, 0.02227434143424034, 0.01088807638734579, 0.009675376117229462, -0.008724882267415524, 0.0026646615006029606, -0.02684982493519783, -0.01832815259695053, 0.01037677563726902, -0.011851680465042591, 0.0033726159017533064, -0.046777430921792984, 0.00827257800847292, 0.026836715638637543, -0.0015756901120766997, -0.013752669095993042, -0.027636440470814705, -0.01900988630950451, -0.0008255370776169002, 0.020674889907240868, 0.024634189903736115, 0.009858920238912106, -0.010632425546646118, -0.0002847386058419943, -0.015968304127454758, -0.03057314082980156, 0.008901870809495449, 0.002187775680795312, 0.02792486734688282, 0.000480163493193686, -0.017580866813659668, -0.023375604301691055, -0.002009148243814707, 0.007761277724057436, 0.019875163212418556, 0.02546013705432415, -0.024935726076364517, 0.017331771552562714, -0.01793484389781952, 0.007748167496174574, -0.02286430448293686, -0.0034316121600568295, 0.009649155661463737, 0.020910874009132385, -0.010068684816360474, -0.023467376828193665, 0.008757658302783966, 0.020425794646143913, -0.005424372851848602, -0.004287057090550661, 0.03135975822806358, -0.015837201848626137, 0.008751102723181248, -0.01765952818095684, 0.019167209044098854, 0.009085414931178093, 0.007308973465114832, -0.0010979848448187113, 0.004867186304181814, -0.012972608208656311, 0.016151847317814827, 0.0037888670340180397, -0.017266221344470978, -0.013411802239716053, -0.004486988764256239, -0.011084729805588722, -0.004247726406902075, -0.010160456411540508, -0.007479406893253326, 0.005149057134985924, 0.011156836524605751, 0.017947955057024956, 0.0012749733868986368, -0.016833581030368805, 0.03912103548645973, 0.0018911559600383043, -0.04806223511695862, -0.0123498709872365, -0.00032980513060465455, -0.0006149534019641578, -0.01882634311914444, -0.005945505574345589, -0.01604696549475193, 0.03188416734337807, 0.02223501168191433, 0.010304668918251991, 0.017908623442053795, -0.011261718347668648, 0.018983665853738785, -0.00200095446780324, 0.00581768061965704, -0.024319544434547424, -0.011399376206099987, 0.008462676778435707, 0.010881520807743073, 0.00800381787121296, -0.023244502022862434, -0.030546920374035835, 0.03634165972471237, -0.035161733627319336, 0.005801292601972818, -0.029838966205716133, -0.01693846471607685, 0.01999315619468689, 0.014565506018698215, -0.010894631035625935, -0.006843558978289366, -0.011976228095591068, 0.018052836880087852, 0.019665399566292763, 0.023546038195490837, 0.00919029675424099, -0.00736141437664628, 0.0308877881616354, -0.00033246816019527614, 0.008672441355884075, -0.026338525116443634, 0.011838570237159729, -0.0007501530344597995, 0.002527003875002265, 0.007335193920880556, -0.019612958654761314, -0.0386490635573864, 0.01073730830103159, 0.027688883244991302, 0.019822722300887108, 0.012959497980773449, 0.013909992761909962, 0.027321795001626015, -0.013103711418807507, 0.006915665231645107, 0.02434576489031315, 0.0047983573749661446, -0.02797730825841427, -0.0007583469850942492, -0.015325901098549366, 0.00968193169683218, 0.015928974375128746, 0.018944334238767624, 0.03070424310863018, -0.01282184012234211, -0.012834950350224972, -0.022510327398777008, -0.0020665056072175503, -0.003136631101369858, 0.00561447162181139, 0.03191038966178894, -0.013385581783950329, 0.020229140296578407, 0.007544958032667637, 0.004205117933452129, -0.0009308289154432714, -0.011628806591033936, 0.01528657041490078, 0.014735938981175423, -0.022313673049211502, 0.01373955886811018, -0.6242584586143494, -0.01873457059264183, -0.006987771950662136, -0.03125487640500069, 0.015273460187017918, 0.019848942756652832, 0.007112319581210613, 0.03505685180425644, 0.0014847376151010394, -0.06880267709493637, -0.010212897323071957, -0.005512867122888565, 0.011222387664020061, -0.015522555448114872, -0.008901870809495449, -0.0021959696896374226, 0.006092996336519718, 0.014617946930229664, 0.02913101203739643, 0.030022509396076202, -0.030809124931693077, 0.027321795001626015, 0.001986205345019698, 0.005850456189364195, 0.011169946752488613, 0.017292441800236702, -0.020229140296578407, -0.028763923794031143, -0.03581724688410759, 0.004621368832886219, -0.03448000177741051, -0.004877019207924604, 0.01143215224146843, 0.0021582776680588722, 0.0378362275660038, -0.008056258782744408, -0.03191038966178894, 0.028475498780608177, -0.0012167965760454535, 0.046462785452604294, 0.012487528845667839, 0.001133218640461564, 0.029576759785413742, -0.008128365501761436, 0.008193916641175747, -0.0072499774396419525, 0.047668926417827606, -0.011864790692925453, -0.010507878847420216, -0.0308877881616354, -0.007000882178544998, -0.01717444881796837, -0.01317581720650196, 0.003302148310467601, 0.007099209353327751, 0.01684669218957424, 0.04549262300133705, 0.00672556646168232, 0.011268273927271366, -0.0066895135678350925, 0.010357110761106014, 0.030651802197098732, 0.005876676645129919, -0.007885824888944626, -0.010448882356286049, 0.009629490785300732, -0.011628806591033936, 0.013195483013987541, 0.029786525294184685, -0.017266221344470978, 0.014749049209058285, 0.00849545281380415, -0.020111149176955223, 0.012828395701944828, -0.006214266177266836, 0.002394262468442321, 0.034532442688941956, -0.00541126262396574, 0.028606601059436798, 0.010199787095189095, -0.01936386339366436, -0.005784905049949884, -0.04669876769185066, 0.006571521051228046, -0.008600334636867046, -0.0057980152778327465, -0.013936213217675686, -0.0432114377617836, 0.00729586323723197, 0.012258099392056465, -0.0037495361175388098, 0.0036970949731767178, 0.006151992361992598, -0.010711087845265865, 0.0015543859917670488, 0.00862655509263277, -0.008364349603652954, 0.008351239375770092, -0.027531558647751808, -0.024542419239878654, 0.00036237595486454666, -0.014316410757601261, -0.03128109499812126, -0.027951087802648544, 0.025381475687026978, -0.001440490479581058, -0.010973292402923107, 0.017502205446362495, 0.020281581208109856, -0.028632821515202522, 0.013274144381284714, -0.0030727186240255833, 0.007603954523801804, -0.014998144470155239, -0.023559149354696274, -0.02618120238184929, 0.04237237945199013, -0.001601091236807406, 0.0054735359735786915, -0.00259911036118865, 0.028187071904540062, 0.00974748283624649, 0.017095787450671196, -0.01564054749906063, 0.03392937034368515, -0.0072434223257005215, -0.005906174890697002, -0.013909992761909962, -0.020294692367315292, 0.01873457059264183, -1.0869742254726589e-05, 0.0006788659957237542, 0.0009193574078381062, 0.0033890039194375277, 0.014316410757601261, -0.013274144381284714, 0.01824949122965336, -0.014014874584972858, 0.026954706758260727, -0.01380511000752449, -0.00707298843190074, -0.02829195372760296, 0.012730068527162075, -0.03463732451200485, -0.0017043346306309104, -0.027819985523819923, -0.0018124943599104881, 0.012231878004968166, -0.031307317316532135, -0.007086099125444889, 0.024503087624907494, -0.013057825155556202, -0.01241542212665081, -0.01756775751709938, -0.0022254677023738623, 0.00784649420529604, -0.01793484389781952, 0.0003799928817898035, -0.008036592975258827, 0.013097155839204788, 0.008757658302783966, 0.018800122663378716, -0.009452502243220806, -0.024450646713376045, -0.023952456191182137, -0.0067124562337994576, 0.027819985523819923, 0.01452617533504963, -0.010638981126248837, -0.018052836880087852, -0.006076608318835497, -0.02249721623957157, -0.0014953897334635258, 0.0293145552277565, 0.014054205268621445, 0.02268076129257679, 0.007728502154350281, -0.0020828936249017715, -0.015339011326432228, -0.020360242575407028, 0.004385384265333414, -0.029340775683522224, -0.031936608254909515, 0.0013880494516342878, 0.04176930710673332, -0.0007685893797315657, 0.009655711241066456, 0.015797870233654976, 0.0017043346306309104, 0.013595346361398697, -0.00709265423938632, -0.016636928543448448, -0.0019222927512601018, -0.0024450647179037333, -0.028344396501779556, 0.007898935116827488, 0.01268418226391077, 0.01623051054775715, -0.020006265491247177, 0.0386490635573864, 0.037626463919878006, 0.018406813964247704, 0.029524318873882294, -0.021435284987092018, 0.00876421295106411, -0.03513551503419876, 0.02375580184161663, -0.02173682115972042, 0.017816850915551186, -0.003500441089272499, 0.007577734068036079, -0.0242933239787817, 0.02466041035950184, -0.0039822435937821865, -0.020727330818772316, 0.015706099569797516, -0.014552395790815353, -0.01506369560956955, -0.007617064751684666, -0.001820688252337277, 0.02034713327884674, 0.03602701053023338, 0.019665399566292763, -0.012067999690771103, -0.007505627349019051, -0.0036184333730489016, 0.011045399121940136, 0.02675805427134037, 0.009183741174638271, -0.020412685349583626, -0.01587653160095215, -0.0034185019321739674, -0.011976228095591068, 0.023362495005130768, 0.004585315473377705, -0.010114570148289204, 0.02371647208929062, -0.010684867389500141, 0.021894143894314766, -0.014198418706655502, -0.0169909056276083, 0.010344000533223152, 0.024988166987895966, -0.010593094862997532, 0.0026154981460422277, -0.010022798553109169, 0.00707298843190074, 0.015076805837452412, -0.028318174183368683, 0.016466494649648666, 0.0008882205584086478, -0.010560319758951664, -0.016243619844317436, 0.005417817737907171, 0.023454265668988228, -0.03298543021082878, 0.02195969596505165, -0.0002236939180875197, -0.03371960297226906, 0.019927604123950005, 0.014971924014389515, 0.028003528714179993, -0.0012495722621679306, 0.007610509637743235, 0.006283095106482506, 0.01604696549475193, 0.017738189548254013, 0.011681247502565384, 0.028187071904540062, -0.010324334725737572, 0.0023844295646995306, -0.016505824401974678, 0.020897764712572098, -0.000393308000639081, 0.01436885166913271, -0.0008570836507715285, 0.007603954523801804, 0.000343325111316517, 0.024948837235569954, -0.009308288805186749, 0.01023911777883768, -0.06859291344881058, 0.04114001616835594, 0.008311908692121506, -0.027819985523819923, -0.01604696549475193, -0.021356623619794846, 0.0028301787097007036, -0.03185794875025749, 0.026246752589941025, 0.009865475818514824, -0.0038576957304030657, -0.006732121575623751, -0.0028531216084957123, 0.00448371097445488, -0.016060076653957367, -6.66779960738495e-05, -0.01717444881796837, -0.037678904831409454, -0.018079057335853577, -0.0204913467168808, -0.01623051054775715, -0.0008783878292888403, -0.04932082071900368, 0.004729528445750475, 0.027898646891117096, 0.00034578327904455364, -0.0066010188311338425, 0.021487725898623466, 0.007630174979567528, 0.007603954523801804, 0.01995382457971573, -0.014172198250889778, 0.0060405549593269825, -0.015548775903880596, 0.014644167385995388, -0.015430782921612263, 0.0010021160123869777, 0.026417186483740807, 0.0285017192363739, -0.021173080429434776, -0.025355255231261253, -0.02842305786907673, -0.0023778744507580996, 0.02653517946600914, 0.027138251811265945, -0.017462873831391335, -0.006623961962759495, 0.01591586321592331, -0.00588323175907135, -0.012022114358842373, -0.030809124931693077, 0.00709265423938632, -0.007794053293764591, 0.017279330641031265, -0.004604980815201998, 0.003467665519565344, 0.022156350314617157, 0.01051443349570036, 0.008030038326978683, 0.0040117413736879826, -0.011871346272528172, 0.02918345294892788, -0.04363096505403519, -0.035974569618701935, 0.0007743251044303179, 0.009498387575149536, 0.0027498784475028515, -0.03114999271929264, -0.03101889044046402, 0.02340182475745678, -0.0019091825233772397, 0.008364349603652954, -0.023231392726302147, 0.003815087489783764, -0.013169262558221817, 0.008239801973104477, -0.016348501667380333, -0.0029907794669270515, -0.0317530632019043, -0.004382106475532055, 0.013424912467598915, -0.013057825155556202, -0.0043296655640006065, 0.018393702805042267, 0.016466494649648666, 0.008252912200987339, -0.009963802061975002, 0.008980532176792622, -0.004182174801826477, 0.010763528756797314, 0.009386950172483921, -0.0005944686126895249, -0.018052836880087852, 0.017502205446362495, -0.015391452237963676, 0.00149293162394315, 0.012769399210810661, -0.0002552405057940632, 0.01680736057460308, 0.02949809841811657, -0.00878387875854969, 0.009668821468949318, 0.009550829418003559, -0.01945563592016697, 0.003161212895065546, 0.003454555058851838, -0.0020763385109603405, -0.0048901294358074665, -0.025001278147101402, -0.016335392370820045, -0.03204149007797241, -0.009649155661463737, 0.02344115637242794, -0.01351668406277895, -0.01573232002556324, 0.0035561597906053066, -0.004503376316279173, 0.05099893733859062, 0.026168091222643852, 0.03542393818497658, -0.00033369724405929446, -0.005696410778909922, 0.019075436517596245, 0.0037233156617730856, -0.035922128707170486, -0.010298114269971848, -0.014683498069643974, -0.0001534310868009925, 0.019245870411396027, -0.008862540125846863, 0.0096622658893466, -0.01801350526511669, 0.014395072124898434, -0.0019222927512601018, 0.002567973453551531, 0.03487330675125122, 0.004264113958925009, 0.003408669261261821, -0.007453186437487602, 0.022929856553673744, 0.02088465355336666, 0.008508563041687012, -0.002520448761060834, 0.016663148999214172, -0.03358850255608559, -0.012769399210810661, -0.005581696052104235, 0.01008834969252348, -0.010035908780992031, -0.012959497980773449, 0.014316410757601261, -0.00044492966844700277, 0.010167011059820652, -0.011504258960485458, -0.02501438744366169, 0.007695726118981838, -0.0032349582761526108, 0.0011217471910640597, 0.01402798481285572, 0.0026695779524743557, 0.014552395790815353, -0.01685980148613453, -0.016964685171842575, 0.023192061111330986, -0.0038052548188716173, 0.0323561355471611, 0.0393570177257061, -0.01855102740228176, 0.01860346831381321, -0.017908623442053795, -0.04433891922235489, -0.0047918022610247135, -0.004015019163489342, -0.01088807638734579, -0.003926524892449379, -0.005886509548872709, -0.003978965803980827, -0.020688001066446304, -0.02164505049586296, -0.02290363423526287, -0.002205802360549569, -0.010035908780992031, -0.006007779389619827, 0.005729186348617077, -0.01317581720650196, -0.004867186304181814, -0.003661041846498847, -0.02976030483841896, -0.02425399236381054, -0.004752471577376127, -0.012310540303587914, -0.0025040607433766127, 0.024267103523015976, 0.02177615277469158, 0.010422661900520325, 0.0017748023383319378, -0.009845810011029243, 0.006197878159582615, -0.021133748814463615, -0.011628806591033936, -0.002854760503396392, -0.013418357819318771, 0.004899961873888969, 0.03786244988441467, -0.002313961973413825, 0.022208791226148605, 0.013123376294970512, -0.008515117689967155, -0.008462676778435707, -0.009242737665772438, -0.021723711863160133, -0.023559149354696274, 0.01971784047782421, 0.011543589644134045, 0.0008759296615608037, 0.01954740658402443, 0.01001624297350645, 0.013221703469753265, -0.016387833282351494, 0.00284001138061285, 0.013923102989792824, -0.022169459611177444, -0.017895514145493507, -0.030809124931693077, -0.020202919840812683, 0.002446703379973769, 0.006129049696028233, -0.03683984652161598, -0.018524806946516037, 0.010776638984680176, -0.013084045611321926, -0.013365915976464748, -0.02357225865125656, 0.048429325222969055, -0.01828882098197937, 0.024503087624907494, -0.008364349603652954, 0.02393934689462185, 0.0018616578308865428, 0.0033496730029582977, -0.023637810721993446, -0.02716447226703167, 0.04116623476147652, 0.024975057691335678, 0.003026832826435566, -0.014513065107166767, -0.004955680575221777, 0.012579300440847874, 0.010763528756797314, -0.00791204534471035, -0.0009234543540515006, -0.010612760670483112, 0.010285004042088985, -0.05773761123418808, -0.024135999381542206, 0.013870662078261375, -0.014735938981175423, 0.0019042661879211664, 0.005093338433653116, -0.02223501168191433, 0.017869293689727783, -0.014565506018698215, -0.019966935738921165, -0.0012970969546586275, 0.0049196272157132626, 0.03586968779563904, 0.011065064929425716, 0.027374235913157463, 0.029550539329648018, 0.0023844295646995306, -0.009563939645886421, 0.020019376650452614, 0.004434547387063503, -0.02125174179673195, 0.01595519483089447, 0.004952403251081705, -0.032880548387765884, -0.016702478751540184, 0.004952403251081705, 0.003916691988706589, -0.02823951281607151, -0.00847578700631857, 0.028396837413311005, 0.004844243172556162, -0.014421292580664158, -0.0007263906882144511, -0.008960867300629616, -0.00793171115219593, 0.0086986618116498, 0.0016994182951748371, 0.0013839524472132325, -0.012985718436539173, -0.01895744539797306, -0.022916745394468307, -0.019796501845121384, -0.013949323445558548, 0.03510929271578789, -0.003467665519565344, -0.033011648803949356, -0.00694844126701355, -0.010049019008874893, 0.007603954523801804, -0.01878701150417328, -0.018944334238767624, 0.025604350492358208, 0.004418159835040569, 0.0029301445465534925, 0.025748562067747116, 0.007302418351173401, 0.025394584983587265, 0.005060562863945961, 0.014604836702346802, 0.024057338014245033, -0.008973977528512478, 0.005771794822067022, -0.0010652091586962342, 0.021907255053520203, -0.018406813964247704, 0.02340182475745678, 0.010606205090880394, 0.010488213039934635, 0.008311908692121506, -0.01617806777358055, 0.010992958210408688, 0.03720693662762642, -0.029681643471121788, -0.004958958365023136, -0.019704729318618774, 0.003992076031863689, -0.03340495750308037, -0.022064577788114548, 0.031936608254909515, 0.04617435857653618, -0.05584973469376564, -0.018000395968556404, -0.02325761318206787, 0.010966737754642963, -0.027321795001626015, 0.03025849536061287, 0.030337156727910042, 0.02627297304570675, -0.010730752721428871, -0.011117505840957165, -0.019914494827389717, 0.019783392548561096, -0.013870662078261375, 0.02173682115972042, 0.009937581606209278, -0.013057825155556202, 0.03993387147784233, -0.02994384802877903, -0.017449764534831047, -0.006361756473779678, 0.007459741551429033, 0.005057285074144602, 0.007571178954094648, 0.002040285151451826, -0.002361486665904522, -0.004047794733196497, 0.008488897234201431, 0.012893946841359138, -0.009944137185811996, 0.02711203135550022, -0.008770768530666828, 0.0025302814319729805, 0.028213292360305786, -0.025394584983587265, -0.013595346361398697, 0.004575483035296202, -0.008659331128001213, -0.011891011148691177, -0.03057314082980156, -0.021946584805846214, -0.007315528579056263, 0.020006265491247177, -0.011818905360996723, -0.03943568095564842, -0.02084532380104065, 0.005355543922632933, -0.01617806777358055, -0.005214608274400234, 0.01591586321592331, -0.006528912577778101, -0.013778889551758766, 0.029262114316225052, 0.006391254719346762, 0.002094364957883954, 0.010160456411540508, 0.006227376405149698, 0.011340380646288395, -0.0052375514060258865, -0.020111149176955223, -0.008823209442198277, -0.024267103523015976, 0.029078571125864983, 0.018275711685419083, -0.023060958832502365, -0.02657450921833515, 0.028187071904540062, -4.151370831095846e-06, 0.009891696274280548, -0.03492574766278267, 0.03411291167140007, 0.036918509751558304, -0.0037036503199487925, 0.013136486522853374, 0.011838570237159729, 0.0073941899463534355, 0.00325626228004694, 0.007426965981721878, 0.030048729851841927, -0.013870662078261375, -0.019022995606064796, 0.00532276788726449, 0.02658762037754059, -0.00954427383840084, -0.004015019163489342, -0.0011758269974961877, 0.01064553577452898, -0.005047452636063099, 0.02262832038104534, -0.010619315318763256, -0.016754919663071632, -0.01632228121161461, -0.0023401824291795492, 0.04800979420542717, 0.0017567756585776806, -0.0008972338400781155, -0.021369734779000282, -0.010140790604054928, -0.006837003864347935, 0.024988166987895966, -0.022510327398777008, 0.01057998463511467, 0.03744291886687279, -0.017161337658762932, 0.006778007838875055, -0.023231392726302147, -0.002127140760421753, 0.021579498425126076, 0.001820688252337277, 0.028029749169945717, 0.005296547431498766, 0.0002814610197674483, -0.019626067951321602, 0.0017895513447001576, 0.0008931368938647211, 0.012592410668730736, 0.008010372519493103, -0.011438706889748573, 0.005090060643851757, -0.01838059350848198, -0.025827225297689438, 0.0032267642673105, -0.023860683664679527, -0.005843901075422764, 0.006509247235953808, -0.006122494116425514, -0.026954706758260727, -0.007748167496174574, -0.04790491238236427, 0.04730184003710747, 0.01239575631916523, -0.01418530847877264, 0.00995069183409214, -0.027243133634328842, -0.017816850915551186, 0.020517567172646523, -0.010468547232449055, -0.021631939336657524, -0.015155468136072159, -0.021225521340966225, 0.029838966205716133, -0.002848205389454961, -0.005375209264457226, -0.02555190958082676, -0.02021603100001812, 0.003320174990221858, -0.002779376460239291, 0.22423799335956573, -0.014224639162421227, 0.0026777719613164663, 0.009708152152597904, 0.015365231782197952, 0.026141870766878128, 0.02523726224899292, -0.003434889717027545, -0.007099209353327751, -0.004857353866100311, -0.02466041035950184, 0.0045427074655890465, -0.013228259049355984, 0.01113717071712017, -0.022733202204108238, -0.04195285215973854, -0.02690226584672928, -0.011733688414096832, -0.008502007462084293, -0.029681643471121788, -0.0018731292802840471, -0.004621368832886219, 0.004303445108234882, -0.02711203135550022, 0.020202919840812683, 0.030022509396076202, -0.013228259049355984, -0.0010791388340294361, 0.034322675317525864, 0.007610509637743235, 0.002795764245092869, -0.010861855931580067, -0.00518838781863451, -0.0035692700184881687, 0.0006874696118757129, -0.005499756895005703, 0.006446973420679569, -0.007636730093508959, 0.00505400775000453, 0.01977028138935566, 0.01587653160095215, 0.010717642493546009, -0.004801635164767504, 0.005250661633908749, -0.013791999779641628, -0.01205488946288824, -0.005634136963635683, -0.03322141617536545, -0.004991733934730291, 0.02115996927022934, -0.015614327043294907, 0.0024762016255408525, 0.010180121287703514, 0.03849174082279205, 0.004460768308490515, -0.0016691007185727358, -0.014159087091684341, -0.013660897500813007, -0.023323163390159607, -0.003208737587556243, 0.00456237280741334, 0.030546920374035835, -0.0007448269752785563, 0.002743323100730777, -0.030861565843224525, 0.023506706580519676, -0.029655423015356064, 0.00988514069467783, 0.007335193920880556, -0.04329010099172592, 0.020596228539943695, -0.0003087058139499277, 0.002928505651652813, 0.00561447162181139, -0.0285017192363739, -0.00820047128945589, -0.012199102900922298, -0.0127562889829278, 0.01752842590212822, 0.014447513036429882, -0.0013905075611546636, -0.019193429499864578, 0.005293270107358694, 0.029157232493162155, -0.0026318859308958054, -0.023205170407891273, 0.01586342230439186, -0.00786616001278162, -0.02325761318206787, -0.0018157719168812037, 0.017607087269425392, 0.007308973465114832, -0.017685748636722565, -0.004470600746572018, -0.006555133033543825, 0.019350752234458923, 0.021081307902932167, 0.047668926417827606, 0.020897764712572098, -0.014460623264312744, -0.037364259362220764, -0.013765779323875904, 0.006545300595462322, -0.012507193721830845, -0.03990764915943146, 0.017738189548254013, -0.0027302128728479147, 0.02612876147031784, 0.02648273855447769, 0.007184425834566355, -0.003713482990860939, -0.013136486522853374, 0.0039986311458051205, -0.005031064618378878, 0.008161140605807304, -0.0065780761651694775, 0.002045201603323221, 0.0181052777916193, 0.0022303841542452574, -0.013936213217675686, 0.027636440470814705, -0.004863908980041742, -0.01873457059264183, -0.007872714661061764, 0.01654515601694584, -0.02666628174483776, -0.0031317148823291063, 0.005755406804382801, 0.00917063094675541, -0.032303694635629654, 0.016781140118837357, -0.014644167385995388, 0.03649898245930672, -0.02855416014790535, -0.012389201670885086, 0.015194798819720745, 0.020412685349583626, 0.003801977261900902, -0.0026253308169543743, 0.006201155949383974, 0.010927407070994377, 0.007453186437487602, 0.017384212464094162, 0.012041779235005379, 0.00793171115219593, -0.04079914838075638, 0.006056942977011204, -0.01623051054775715, -0.015142357908189297, -0.015037475153803825, -0.0073941899463534355, -0.03804599121212959, -0.0074990722350776196, -0.009760593064129353, 0.028213292360305786, -0.011969673447310925, -0.007518737576901913, -0.017331771552562714, 0.014893262647092342, -0.004221505951136351, -0.03948812186717987, -0.004981901030987501, 0.023952456191182137, -0.011051954701542854, -0.0008824847755022347, 0.01156325452029705, -0.16497959196567535, 0.023113399744033813, 0.001533901202492416, -0.025263482704758644, 0.021671270951628685, 0.0012389201438054442, 0.014329520985484123, 0.02766266092658043, -0.0355026014149189, 0.01748909428715706, 0.006876334547996521, -0.014683498069643974, -0.021487725898623466, -0.0009472167585045099, -0.022313673049211502, 0.0017420266522094607, -0.03602701053023338, 0.005837345961481333, -0.009688486345112324, 0.0006104467902332544, 0.020452015101909637, -0.026731831952929497, -0.00028043679776601493, -0.005978281609714031, 0.019560517743229866, -0.007276197895407677, 0.0017453042091801763, 0.021854814141988754, -0.016610708087682724, -0.029104791581630707, -0.015850311145186424, 0.013975543901324272, 0.0030169999226927757, 0.00484096584841609, 0.031936608254909515, -0.015981415286660194, 0.014172198250889778, -0.022838084027171135, -0.008443011902272701, 0.018236380070447922, 0.011891011148691177, 0.030048729851841927, 0.03209393098950386, 0.009878586046397686, 0.0005944686126895249, 0.030284715816378593, 0.011995893903076649, -0.006063498090952635, 0.006889444775879383, -0.008226691745221615, -0.0016461578197777271, -0.01465727761387825, 0.029419437050819397, 0.009668821468949318, 0.013791999779641628, 0.027321795001626015, 0.010684867389500141, 0.01936386339366436, -0.004903239663690329, -0.0015666767721995711, -0.01257274579256773, -0.02944565750658512, -0.010167011059820652, 0.004332942888140678, 0.019875163212418556, -0.0075908442959189415, -0.012907057069242, -0.01640094257891178, -0.02684982493519783, -0.003647931618615985, 0.0007919420022517443, -0.01671558991074562, 0.023152729496359825, -0.02492261677980423, 0.0032185702584683895, -0.0046639773063361645, -0.021356623619794846, 0.005280159879475832, 0.0068107834085822105, 0.0046803648583590984, 0.007328638806939125, 0.030546920374035835, -0.035738587379455566, -0.0070533230900764465, 0.011714022606611252, 0.010566874407231808, -0.002133695874363184, -0.008875650353729725, 0.007761277724057436, -0.01494570355862379, 0.010114570148289204, -0.016741810366511345, -0.008429900743067265, -0.034401338547468185, 0.015129247680306435, 0.03277566656470299, 0.005663635209202766, 0.02653517946600914, 0.0052473838441073895, 0.0016092851292341948, 0.014788379892706871, 0.00018948431534226984, -0.032434798777103424, 0.017292441800236702, 0.019036106765270233, 0.000306862173601985, -0.014460623264312744, 0.018131498247385025, 0.037678904831409454, 0.0016641843831166625, -0.014565506018698215, 0.003225125605240464, 0.034453779458999634, 0.03448000177741051, -0.026325413957238197, 0.010134235955774784, -0.01185823604464531, -0.024857064709067345, 0.006342091131955385, -0.00603072252124548, 0.054119180887937546, 0.0024909505154937506, -0.003700372762978077, 0.004496821202337742, -0.02483084425330162, -0.003328368766233325, -0.09266336262226105, 0.013726448640227318, -0.010265338234603405, 0.0028645931743085384, 0.0277937650680542, 0.0013724809978157282, 0.006548577919602394, 0.0009423003648407757, -0.00350699620321393, 0.010285004042088985, -0.01649271510541439, -0.034322675317525864, 0.00019562974921427667, 0.0015158745227381587, 0.017200669273734093, 0.0001893818989628926, 0.015024364925920963, -0.024896396324038506, -0.0035332166589796543, 0.01418530847877264, -0.009498387575149536, 0.010455437004566193, 0.008141475729644299, -0.0015617604367434978, -0.013392137363553047, -0.017725080251693726, -0.02667939104139805, 0.01277595479041338, 0.012428532354533672, 0.007512182462960482, 0.010285004042088985, 0.0029956959187984467, -0.009131300263106823, 0.007367969490587711, -0.024306433275341988, -0.0001911230938276276, -0.014827711507678032, -0.006151992361992598, 0.024857064709067345, -0.03922591730952263, 0.021395955234766006, -0.004647589288651943, 0.01198933832347393, -0.01192378718405962, 0.024909505620598793, 0.020989535376429558, -0.022208791226148605, 0.009839254431426525, -0.012874281033873558, -0.022287452593445778, -0.03639410063624382, -0.015548775903880596, 0.00048180227167904377, -0.019888274371623993, 0.03615811467170715, 0.03128109499812126, -0.011910676956176758, 0.01608629710972309, 0.002200885908678174, 0.005178555380553007, 0.001552747213281691, -0.0006809144397266209, -0.01927209086716175, 0.0033890039194375277, 0.010750418528914452, 0.0011414125328883529, 0.0036577642895281315, -0.006161825265735388, 0.033195193856954575, 0.0031153270974755287, 0.00707298843190074, 0.021907255053520203, -0.01078319363296032, -0.012238433584570885, -0.009799923747777939, 0.00990480650216341, -0.024843955412507057, 0.00907230470329523, -0.003100577974691987, -0.020189810544252396, -0.0053358785808086395, -0.01600763574242592, -0.0017502205446362495, -0.029917627573013306, 0.0012921806192025542, 0.011668137274682522, -0.00143967114854604, 0.011569810099899769, 0.00862655509263277, -0.031228654086589813, 0.013700228184461594, 0.023414935916662216, -0.01345113292336464, -0.007728502154350281, -0.018406813964247704, 0.034532442688941956, -0.000706315622664988, -0.008456122130155563, 0.022431666031479836, 0.02608942985534668, -0.006928775925189257, 0.018852563574910164, -0.07913357019424438, 0.018852563574910164, -0.009472167119383812, -0.0018682129448279738, 0.002681049518287182, -0.007662950549274683, 0.008600334636867046, 0.007197536062449217, 0.0020828936249017715, 0.0022975741885602474, -0.021592607721686363, 0.019298311322927475, 0.014552395790815353, 0.01587653160095215, -0.03227747604250908, -0.032251253724098206, 0.011366601102054119, 0.012356425635516644, 0.011779574677348137, 0.007610509637743235, -0.015089916065335274, -1.1906785175597179e-06, 0.004657422192394733, 0.006001224275678396, 0.013477353379130363, 0.02416221983730793, -0.011202722787857056, 0.022379225119948387, -0.025001278147101402, -0.006876334547996521, 0.0043526082299649715, -0.026797384023666382, -0.022759422659873962, 0.014237749390304089, -0.011884456500411034, -0.026797384023666382, 0.0003550014225766063, -0.01129449438303709, 0.04447002336382866, 0.01801350526511669, -0.019154099747538567, -0.014565506018698215, 0.027138251811265945, -0.04155954346060753, -0.004467323422431946, 0.011995893903076649, -0.012074555270373821, 0.007735057268291712, 0.04908483847975731, -0.020923985168337822, 0.021055087447166443, 0.035292837768793106, -0.01004246436059475, -0.021448396146297455, 0.022470995783805847, -0.025158600881695747, 0.02058311738073826, -0.00939350575208664, 0.017043346539139748, -0.052467286586761475, 0.01214666198939085, -0.009367285296320915, 0.019075436517596245, -0.01814460940659046, 0.02546013705432415, 0.00427066907286644, 0.017462873831391335, -0.017961064353585243, 0.017187559977173805, -0.027059590443968773, 0.0009054277907125652, -0.0030055285897105932, -0.00203864648938179, 0.013975543901324272, -0.007617064751684666, 0.03125487640500069, 0.010894631035625935, -0.0015117776347324252, -0.037233155220746994, 0.01788240298628807, -0.008056258782744408, -0.005076950415968895, -0.01832815259695053, 0.004178897477686405, 0.04116623476147652, 0.007374524604529142, 0.006184767931699753, 0.004503376316279173, -0.007472851779311895, 0.008836319670081139, -0.005834068637341261, 0.026377854868769646, -0.012421977706253529, -0.03322141617536545, -0.006676402874290943, -0.007833383977413177, -0.027400456368923187, -0.0007616245420649648, 0.01841992326080799, 0.00898708775639534, 0.028003528714179993, 0.00707298843190074, -0.0020353689324110746, -0.02608942985534668, -0.00905263889580965, 0.02192036435008049, 0.00638797739520669, -0.032434798777103424, 0.026876045390963554, 0.005024509504437447, 0.0036380989477038383, 0.014290190301835537, 0.01834126189351082, 0.011812349781394005, -0.012244988232851028, 0.013084045611321926, -0.013949323445558548, -0.02192036435008049, -0.03146464005112648, 0.02644340693950653, 0.01608629710972309, 0.014788379892706871, 0.002607304137200117, -0.0019288478652015328, 0.018537916243076324, -0.004886851646006107, 0.00150768063031137, -0.024896396324038506, -0.005414539948105812, -0.015968304127454758, -0.008128365501761436, -0.001779718673788011, -0.0035692700184881687, -0.016335392370820045, 0.00532276788726449, 0.01037677563726902, -0.013254479505121708, 0.02886880561709404, 0.002854760503396392, 0.08647531270980835, 0.0017813574522733688, -0.014329520985484123, 0.009931026957929134, -0.0023713193368166685, -0.006696068681776524, -0.0028121520299464464, -0.0005256397416815162, -0.022536547854542732, -0.009452502243220806, 0.023008517920970917, -0.0014232832472771406, 0.008554448373615742, -0.018537916243076324, -0.02627297304570675, 0.01165502704679966, -0.002141889650374651, 0.020923985168337822, -0.0370233915746212, -0.01667625829577446, 0.024450646713376045, -0.007138540036976337, 0.008882205002009869, -0.012598966248333454, 0.006417475175112486, -0.00954427383840084, 0.019691620022058487, -0.011150281876325607, -0.005037619732320309, 0.0036708745174109936, 0.017423544079065323, -0.016112517565488815, 0.00134789920412004, -0.009439392015337944, -0.00017360859783366323, -0.00259911036118865, -0.013228259049355984, -0.0003347214951645583, 0.012828395701944828, -0.007118874695152044, -0.005863566417247057, 0.04339498281478882, -0.016479603946208954, -0.025853445753455162, 0.026928486302495003, -0.0033168974332511425, -0.006663292646408081, 0.0038773613050580025, -0.002998973475769162]\",\n          \"[-0.00482430774718523, -0.008132588118314743, 0.007914177142083645, -0.03255604952573776, -0.0026482301764190197, 0.03484293818473816, 0.011890538036823273, -0.027082931250333786, -0.011627160012722015, -0.00856941007077694, 0.0011611100053414702, 0.001787434914149344, 0.01910451613366604, -0.014684909954667091, 0.0088970260694623, 0.001922335708513856, 0.04252585768699646, -0.0001895034365588799, 0.030192073434591293, -0.034200552850961685, 0.01302113477140665, -0.0001608972524991259, -0.01061861589550972, -0.007946296595036983, -0.012031862512230873, -0.00043802594882436097, 0.008620800450444221, -0.01667630299925804, 0.025091540068387985, -0.031810883432626724, 0.01002119854092598, 0.004207618534564972, 0.021853921934962273, -0.01861630380153656, -0.04550652205944061, -0.036821480840444565, 0.0015144215431064367, -0.003234406001865864, -0.011697822250425816, -0.01002119854092598, 0.017460010945796967, -0.011813451535999775, -0.01373418141156435, -0.0212115366011858, -0.028136443346738815, 0.007804972119629383, -0.02063339203596115, -0.016445044428110123, -0.028676046058535576, 0.010464443825185299, 0.010310271754860878, 0.0313226692378521, -0.011318815872073174, -0.01784544251859188, -0.014068221673369408, 0.017549945041537285, 0.010952656157314777, 0.024294983595609665, 0.012218154035508633, 0.004792188759893179, 0.0071818591095507145, -0.016046766191720963, -0.02339564450085163, -0.023280015215277672, -0.0010776000563055277, 0.041575126349925995, -0.007965568453073502, -0.021725444123148918, -0.01359285693615675, -0.01798676699399948, 0.03193935751914978, 0.00876854918897152, -0.0013024345971643925, -0.009205370210111141, 0.052932485938072205, -0.012468683533370495, 0.005286824889481068, -0.022984517738223076, -0.017511403188109398, -0.013939744792878628, 0.012365901842713356, -0.011325239203870296, -0.034431811422109604, 0.006362818647176027, 0.021301470696926117, 0.010252457112073898, 0.004236525855958462, 0.043682146817445755, -0.027314189821481705, 0.010901265777647495, -0.021738292649388313, 0.04357936605811119, -0.010554377920925617, -0.0004356170247774571, 0.006086593493819237, 0.006276096682995558, -0.003187833121046424, 0.019913921132683754, 0.010098285041749477, 0.004101625178009272, -0.011293119750916958, -0.017408620566129684, -0.0026562600396573544, -0.013001862913370132, -0.023755380883812904, -0.006147619802504778, -0.006815699860453606, -0.0007857165765017271, 0.020235111936926842, -0.025284254923462868, -0.003070598002523184, 0.049232348799705505, 0.005575897637754679, -0.01753709837794304, -0.008042654022574425, -0.007740733679383993, 0.007515898905694485, -0.022277895361185074, 0.0015674182213842869, -0.008061925880610943, -0.00205241865478456, 0.01372133381664753, 0.00969358254224062, 0.019348621368408203, 0.016316566616296768, 0.012520074844360352, -0.016162395477294922, -0.002746193902567029, -0.027416972443461418, -0.008421661332249641, 0.03592214360833168, 0.024577632546424866, -0.00906404573470354, -0.02212372235953808, -0.019682662561535835, 0.03941671550273895, -0.02322862483561039, 0.004933513235300779, -0.015609944239258766, -0.01728014461696148, 0.021982397884130478, 0.007554442156106234, -0.008203250356018543, -0.03802916407585144, -0.00873642973601818, 0.025464123114943504, 0.03348108008503914, -0.00026478286599740386, -0.0004127320717088878, -0.020286504179239273, 0.038440290838479996, -0.015609944239258766, 0.01582835428416729, -0.014222393743693829, 0.03204214200377464, -0.0014060190878808498, 0.00105511664878577, 0.010785636492073536, -0.023665446788072586, -0.004120896570384502, -0.010368086397647858, 0.023280015215277672, 0.02641485258936882, 0.006401361897587776, -0.003748313756659627, 0.025348493829369545, -0.0067450376227498055, -0.00022463384084403515, 0.017254449427127838, -0.0023752166889607906, -0.01313676405698061, 0.01796107180416584, -0.03556240722537041, 0.011672127060592175, 0.0026161109562963247, 0.0019656966906040907, 0.0088263638317585, -0.010001927614212036, -0.01516027469187975, -0.02465471811592579, -0.015815507620573044, 0.00127754220739007, 0.005800732411444187, 0.01384981069713831, -0.0009274426847696304, -0.00471189059317112, 0.005710798315703869, 0.017010342329740524, -0.0021969550289213657, 0.007496627513319254, 0.010239609517157078, 0.0017023190157487988, -0.026594718918204308, -0.026080811396241188, -0.6319007873535156, -0.020003855228424072, 0.010856298729777336, -0.022444915026426315, 0.010708549991250038, 0.010136827826499939, 0.015468619763851166, 0.02479604259133339, -0.012160339392721653, -0.04483843967318535, -0.03222200646996498, 0.007201130501925945, 0.010509410873055458, 0.0010832209372892976, -0.015018950216472149, -0.007804972119629383, 0.006860666908323765, 0.013939744792878628, 0.02954968810081482, 0.00038964636041782796, -0.012108948081731796, 0.02496306225657463, -8.064936992013827e-05, -0.007959144189953804, 0.00971927773207426, 0.005925997160375118, 0.005640136078000069, -0.024333525449037552, -0.03006359562277794, 0.00403417507186532, -0.029703861102461815, 0.02160981483757496, 0.003719406435266137, -0.002521359361708164, 0.05195606127381325, -0.027288494631648064, -0.012558617629110813, 0.028830217197537422, 0.004991327878087759, 0.031785186380147934, -0.007727886084467173, -0.002906790003180504, 0.016984647139906883, 0.007573713548481464, 0.000542011926881969, -0.015622791834175587, 0.033378299325704575, -0.005585533566772938, -0.005534142721444368, -0.007091925013810396, 0.004634804558008909, -0.03255604952573776, -0.022650478407740593, -0.017010342329740524, -0.0056690433993935585, 0.012038285844027996, 0.026491938158869743, -0.007201130501925945, 0.015982527285814285, -0.011023318395018578, -0.002098991535604, 0.024359220638871193, 0.008935568854212761, 0.017614183947443962, 0.01384981069713831, 0.00479540042579174, -0.0212115366011858, 0.003517055418342352, 0.00964861549437046, -0.0020138754043728113, 0.024102266877889633, -0.0009242307278327644, -0.031271278858184814, 0.011864841915667057, -0.0037900686729699373, 0.01981113851070404, 0.04959208518266678, -0.013245969079434872, 0.033378299325704575, 0.005608017090708017, -0.00997623149305582, -0.0053414274007081985, -0.0274683628231287, -0.015751268714666367, 0.006115500815212727, 0.0023430974688380957, -0.005553414113819599, 0.003745101857930422, -0.018950343132019043, 0.008415237069129944, -0.011460140347480774, 0.021558424457907677, -0.00026859703939408064, 0.015108884312212467, 0.00869788695126772, 0.019310079514980316, -0.005945269018411636, 0.003250465728342533, -0.00043882892350666225, -0.044196054339408875, 0.029935119673609734, -0.014016830362379551, -0.0030111775267869234, -0.031091410666704178, -0.0061797392554581165, 0.01639365404844284, -0.010657159611582756, 0.012661399319767952, 0.015866898000240326, -0.037618037313222885, 0.008916296996176243, -0.013245969079434872, 0.006366030778735876, -0.011794179677963257, -0.00514549994841218, -0.025862401351332664, 0.004275069106370211, 0.009019078686833382, 0.00812616478651762, -0.020003855228424072, 0.0287531316280365, 0.0021696537733078003, -0.007483779918402433, -0.008530866354703903, 0.024616174399852753, -0.014851930551230907, -0.014582129195332527, 0.004692619200795889, -0.01314961165189743, -0.007348879240453243, 0.019425708800554276, 0.0014967559836804867, 0.023691141977906227, -0.01568702980875969, 0.0355881042778492, -0.004898182116448879, 0.03720691055059433, 0.006109077017754316, 0.019374316558241844, -0.01889895275235176, -0.011016895063221455, -0.012449412606656551, 0.019605575129389763, -0.0010446778032928705, -0.007875634357333183, -0.022496305406093597, -0.007213978096842766, -0.005546990316361189, -0.025913791730999947, -0.009597225114703178, -0.003350035287439823, 0.001769769354723394, -0.027879489585757256, -0.001447774120606482, -0.00518725486472249, 0.015442924574017525, -0.019708357751369476, -0.004387486260384321, -0.02402518130838871, -0.0062696728855371475, 0.0028473695274442434, 0.0341491624712944, -0.004435664974153042, -0.003488148096948862, -0.0012141068000346422, -0.019348621368408203, 0.009372389875352383, -0.0029180317651480436, 0.012269544415175915, -0.008113316260278225, 0.0019062760984525084, -0.013952592387795448, 0.0057686129584908485, -0.0031958629842847586, -0.005357487127184868, 0.008312455378472805, -0.007098349276930094, 0.019746899604797363, -0.008286760188639164, 0.005563050042837858, 0.0089676883071661, -0.016046766191720963, -0.02362690307199955, -0.00838311854749918, 0.04627738147974014, -0.009057621471583843, 0.007843514904379845, 0.01847497932612896, 0.005897089838981628, 0.022213656455278397, -0.0018324018456041813, -0.007785700727254152, 0.009083317592740059, 0.005659407936036587, -0.0021792894694954157, 0.01503179781138897, 0.008113316260278225, 0.012597160413861275, -0.02166120707988739, 0.03569088503718376, 0.008659343235194683, -0.007702190428972244, 0.02402518130838871, -0.005219374317675829, 0.00453202286735177, -0.03219631314277649, 0.007361726835370064, -0.0260422695428133, 0.014299479313194752, -0.0053157322108745575, -0.015134579502046108, -0.028444787487387657, 0.001430108561180532, -0.005797520279884338, -0.009430204518139362, 0.03006359562277794, 0.00011773704318329692, 0.004705466795712709, -0.025888096541166306, -0.001327326986938715, 0.007888481952250004, 0.02505299635231495, 0.021083060652017593, 0.012982591055333614, -0.03743816912174225, -0.020376436412334442, 0.018423588946461678, 0.0042686453089118, 0.019053125753998756, -0.011286696419119835, -0.018487825989723206, 0.013811266981065273, 0.0042333141900599, 0.009398085996508598, -0.005383182317018509, -0.006590865086764097, 0.023177234455943108, -0.03227340057492256, 0.03062889538705349, -0.031219888478517532, -0.008235369808971882, 0.007824243977665901, 0.00872358214110136, -0.0024025181774049997, 0.019310079514980316, -0.011151795275509357, 0.01467206235975027, -0.017344381660223007, -0.010085437446832657, 0.015275903977453709, -0.003414273727685213, 0.0024330313317477703, -0.003812552196905017, -0.010271728970110416, 0.021648358553647995, -0.0432710237801075, 0.0046315924264490604, 0.013772724196314812, -0.00883921142667532, 0.021185841411352158, -0.006989143788814545, 0.01289265789091587, 0.0003677651402540505, 0.0015320871025323868, 0.01955418474972248, 0.030783066526055336, 0.012391597963869572, 0.011100404895842075, 0.0033275517635047436, -0.014569281600415707, -0.012481531128287315, -0.02800796553492546, 0.019875377416610718, 0.0012197276810184121, 0.004111261107027531, -0.011068285442888737, -0.010464443825185299, -0.0033275517635047436, 0.007580137345939875, -0.015250208787620068, -0.015250208787620068, -0.04193486273288727, 0.022868888452649117, 0.018423588946461678, -0.027905184775590897, -0.019991006702184677, 0.00017816133913584054, 0.006279308814555407, -0.01571272499859333, 0.01576411724090576, 0.009276032447814941, -0.009205370210111141, 0.00035913308965973556, 0.0008993383380584419, 0.0026594719383865595, 0.005264341365545988, 0.003786856774240732, -0.01360570453107357, -0.01685616932809353, -0.002587203634902835, -0.01688186638057232, -0.02018372155725956, -0.037052739411592484, -0.011370206251740456, -0.0003382556024007499, 0.03409777209162712, -0.01702319085597992, -0.005974176339805126, 0.0011755636660382152, 0.005347851198166609, -0.01824372075498104, 0.013014710508286953, -0.02388385683298111, 0.00504593038931489, -0.0166506078094244, -0.004679771140217781, -0.02800796553492546, -0.003123594680801034, 0.031810883432626724, 0.009443053044378757, -0.021674053743481636, -0.007252521347254515, -0.019001733511686325, -0.014312326908111572, 0.04727950319647789, 0.04915526509284973, -0.03492002189159393, 0.011293119750916958, 0.0303205493837595, 0.0185006745159626, -0.011774908751249313, -0.02117299474775791, 0.03296717256307602, -0.0017328322865068913, 0.02117299474775791, 0.005771825090050697, 0.011595040559768677, 0.018115242943167686, -0.0016846534563228488, 0.0024330313317477703, 0.01587974652647972, -0.019143059849739075, 0.019798291847109795, -0.014543585479259491, -0.018796171993017197, -0.004901393782347441, 0.024603327736258507, -0.003311492269858718, -0.010316696017980576, -0.012809147126972675, 0.017922528088092804, 0.021584119647741318, 0.02308730036020279, -0.01346438005566597, -0.012757756747305393, -0.004021327011287212, 0.010631464421749115, -0.006231130100786686, -0.007393845822662115, -0.028085051104426384, -0.001330539002083242, 0.021648358553647995, 0.009314576163887978, -0.018089547753334045, 0.032607439905405045, 0.015571401454508305, 0.006375666242092848, -0.0177940521389246, 0.011729941703379154, -0.017151666805148125, -0.018153786659240723, 0.012687094509601593, -0.012442988343536854, -0.024731803685426712, 0.027828097343444824, -0.01282199565321207, 0.01662491075694561, -0.010393781587481499, 0.002359157195314765, -0.01289265789091587, 0.033403996378183365, 0.004278280772268772, 6.258230860112235e-05, 0.06305646896362305, -0.004336095415055752, -0.017151666805148125, 0.0073553030379116535, -0.001122566987760365, -0.0012020620051771402, -0.04244877025485039, -0.018397893756628036, -0.021879617124795914, -0.025515513494610786, 0.009982655756175518, -0.011029742658138275, -0.006038414780050516, 0.0025743560399860144, 0.0027622533962130547, 0.028342004865407944, 0.02160981483757496, 0.017922528088092804, -0.01373418141156435, -0.016946103423833847, -0.0008551743812859058, -6.062503962311894e-05, -0.036333270370960236, 0.012667822651565075, -0.0090383505448699, 0.012674246914684772, 0.012603584676980972, 0.020080940797924995, 0.016804778948426247, -0.026825977489352226, -0.005559837911278009, 0.0026658957358449697, 0.016496434807777405, 0.016894713044166565, -0.013098220340907574, 0.01627802476286888, 0.00788205862045288, 0.0020700842142105103, 0.0035331149119883776, 0.009076893329620361, -0.021571272984147072, 0.01491616852581501, -0.0387229397892952, -0.0028345216996967793, -0.011961200274527073, 0.028393397107720375, 0.0016991071170195937, -0.012532922439277172, 0.013502922840416431, -0.009455900639295578, -0.015571401454508305, 0.019746899604797363, -0.02453908883035183, 0.013361598365008831, -0.005620864685624838, 0.0011667308863252401, 0.010965503752231598, -0.00434894347563386, 0.03694995865225792, 0.004075929988175631, -0.026029421016573906, 0.012275968678295612, -0.01796107180416584, 0.025656837970018387, 0.02444915473461151, -0.04021327197551727, -0.0009748185402713716, -0.02573392353951931, -0.0187833234667778, -0.0170745812356472, -0.026569023728370667, 0.0033371876925230026, 0.010213914327323437, 0.0060062953270971775, 0.006211858242750168, -0.016072461381554604, -0.006035202648490667, -0.01453073788434267, -0.006976296193897724, -0.0056433482095599174, 0.005797520279884338, -0.012295239605009556, -0.01555855292826891, -0.005036294925957918, -0.025117235258221626, -0.026748891919851303, -0.025258559733629227, -0.008473051711916924, 0.01361855212599039, 0.0029003662057220936, 0.012057557702064514, 0.009667887352406979, -0.02336994931101799, 0.012719213962554932, -0.006475236266851425, 0.005720434244722128, -0.00957795325666666, 0.007021262776106596, -0.015083189122378826, -0.0006696858908981085, 0.008261064998805523, 0.02821352891623974, -0.0011322026839479804, -0.0036294725723564625, 0.007252521347254515, -0.014171002432703972, -0.0030609623063355684, -0.00852444302290678, -0.015391533263027668, -0.008061925880610943, 0.03088584914803505, 0.0088970260694623, 0.04067578911781311, 0.024436308071017265, 0.005983811803162098, 0.02638915739953518, 0.012944048270583153, 0.00859510526061058, 0.003314704168587923, -0.007053382229059935, -0.034740153700113297, -0.026337765157222748, 0.003924969583749771, 0.009545833803713322, 0.011113252490758896, -0.007169011514633894, -0.002135928487405181, 0.025836706161499023, -0.011517954990267754, -0.007914177142083645, -0.04062439873814583, 0.028598958626389503, -0.027288494631648064, 0.031862273812294006, -0.019207296893000603, 0.010702126659452915, -0.0100340461358428, 0.011472987942397594, -0.012982591055333614, -0.019631272181868553, 0.012166762724518776, 0.0007688539917580783, -0.0015136185102164745, -0.0002531396457925439, -0.005171195603907108, 0.01360570453107357, 0.0019946040119975805, -0.007329607382416725, -0.007580137345939875, 0.02510438673198223, -0.00396993663161993, -0.01924584060907364, -0.0318879671394825, 0.018269415944814682, -0.0034945718944072723, -0.0037579494528472424, 0.011640007607638836, -0.014749148860573769, 0.011241729371249676, 0.003044902579858899, -0.026825977489352226, -0.006340335123240948, 0.00425579771399498, 0.06053831800818443, 0.011736365035176277, 0.01770411804318428, 0.02444915473461151, -0.02308730036020279, -0.01093338429927826, 0.013888353481888771, 0.035819362848997116, -4.27085324190557e-05, 0.02095458284020424, 0.013682790100574493, -0.035742275416851044, -0.012989015318453312, -0.0025599023792892694, -0.013400141149759293, -0.021481338888406754, -0.030705980956554413, 0.043450888246297836, 0.0037418899592012167, 0.001430108561180532, 0.0011964412406086922, 0.016663454473018646, -0.015789812430739403, 0.011113252490758896, -0.01427378412336111, 0.007560865953564644, -0.02414081059396267, -0.01399113517254591, -0.021725444123148918, -0.0003613412845879793, 0.003953876905143261, 0.01704888604581356, 0.006661527790129185, -0.014466499909758568, -0.010907689109444618, -0.02055630460381508, 0.005106957163661718, 0.0026578661054372787, -0.01781974732875824, 0.03782360255718231, -0.017177361994981766, 0.0018532794201746583, 0.03515128046274185, 0.017485706135630608, 0.0019207297591492534, 0.0015722361858934164, -0.028033660724759102, 0.025631142780184746, 0.0002894745266530663, 0.014980407431721687, -0.014286631718277931, 0.00991841685026884, 0.00449669174849987, 0.025798162445425987, 0.007336031179875135, 0.015841202810406685, -0.005797520279884338, -0.01704888604581356, 0.03545962646603584, 0.031219888478517532, -0.02063339203596115, 0.008408813737332821, -0.02280465140938759, 0.013631399720907211, -0.011350934393703938, -0.019232992082834244, 0.02954968810081482, 0.024500545114278793, -0.04835870862007141, -0.03941671550273895, -0.007079077418893576, 0.00991841685026884, -0.03355816751718521, -0.012102524749934673, 0.0023318559397011995, 0.021057365462183952, 0.0032279822044074535, -0.001761739607900381, -0.017627030611038208, 0.01585404947400093, -0.015622791834175587, 0.0293184295296669, 0.006590865086764097, -0.01662491075694561, 0.04257724806666374, -0.026337765157222748, -0.016380805522203445, -0.01611100323498249, 0.019798291847109795, 0.014363718219101429, -0.02234213426709175, 0.0028505814261734486, 0.019207296893000603, 0.007586561143398285, 0.006472024135291576, -0.013258816674351692, -0.00433288374915719, 0.01742146909236908, 0.019284384325146675, 0.0001638080575503409, 0.007779276464134455, -0.01332305558025837, -0.002877882681787014, 0.024731803685426712, -0.013965439982712269, 0.007856362499296665, -0.027879489585757256, 0.020247960463166237, 0.015250208787620068, 0.010843451134860516, 0.01476199645549059, -0.0186419989913702, -0.04062439873814583, -0.013914048671722412, -0.005007387604564428, 0.0032071047462522984, 0.024551937356591225, -0.003953876905143261, 0.000597819103859365, 0.019220145419239998, 0.013888353481888771, -0.015584249049425125, -0.016072461381554604, -0.010862722061574459, -0.005656195804476738, -0.0019335774704813957, -0.05416586250066757, -0.012931200675666332, -0.01004689373075962, 0.027802402153611183, 0.00963576789945364, -0.014453652314841747, -0.022470610216259956, 0.008023383095860481, -0.008730005472898483, 0.02662041410803795, -0.013836963102221489, 0.01674054004251957, 0.021596968173980713, -0.020029550418257713, -0.011363781988620758, 0.01756279356777668, -0.004850003402680159, -0.00018849970365408808, 0.012410868890583515, 0.012141067534685135, -0.0269030649214983, 0.012680670246481895, 0.02479604259133339, 0.011183914728462696, -0.015931136906147003, 0.001527269254438579, -0.009186098352074623, 0.017228752374649048, 0.004509539343416691, 0.0043136123567819595, -0.017575640231370926, 0.001614794135093689, -0.015597096644341946, 0.012237424962222576, 0.039982013404369354, -0.00488212238997221, 0.02308730036020279, -0.004188347142189741, -0.019682662561535835, -0.0015746450517326593, -0.0038318238221108913, -0.02880452200770378, -0.007143315859138966, 0.024269288405776024, -0.007907753810286522, 0.002731740241870284, -0.022650478407740593, -0.015211666002869606, 0.013669942505657673, -0.012667822651565075, 0.030834456905722618, -0.006545898504555225, 0.0024025181774049997, -0.005787884816527367, 0.017203057184815407, 0.0050170230679214, -0.008016958832740784, 0.0003751927288249135, -0.01437656581401825, 0.016843322664499283, -0.01702319085597992, -0.020594848319888115, 0.010625040158629417, -0.02510438673198223, -0.006536262575536966, -0.023588359355926514, -0.01585404947400093, -0.03466306999325752, -0.002486028242856264, -0.023678293451666832, 0.03345538675785065, 0.009404509328305721, -0.0030898696277290583, 0.02015802636742592, -0.038928501307964325, 0.0022114086896181107, 0.007316759787499905, -0.025232864543795586, 0.009160403162240982, 0.00211023329757154, -0.019734052941203117, 0.02365259826183319, 0.0017922528786584735, -0.022046636790037155, -0.0014558039838448167, 3.1542083888780326e-05, 0.0043296716175973415, -0.00511016882956028, 0.23516413569450378, 0.007907753810286522, -0.0005263538332656026, 0.030834456905722618, 0.007361726835370064, 0.014093916863203049, 0.026980150490999222, 0.012153915129601955, 0.0032874029129743576, 0.00919252261519432, -0.023125842213630676, -0.00025193518376909196, -0.04116399958729744, -0.0030786278657615185, 0.003552386537194252, -0.016290871426463127, -0.0395708866417408, -0.022187961265444756, -0.009359542280435562, -0.04332241415977478, -0.013490075245499611, 0.0034014261327683926, -0.012160339392721653, -0.020260808989405632, 0.009603648446500301, 0.035767972469329834, -0.025258559733629227, -0.00883921142667532, 0.017601335421204567, -0.0039763604290783405, 0.007284640800207853, -0.0007250915514305234, -0.01568702980875969, 0.003250465728342533, 0.006549110170453787, 0.003892850363627076, 0.03306995704770088, -0.0022531638387590647, 0.005890666041523218, 0.01611100323498249, 0.022265048697590828, 0.015275903977453709, 0.017575640231370926, 0.003491359995678067, -0.01503179781138897, 0.029780946671962738, -0.011556497775018215, -0.006314639933407307, -0.018487825989723206, 0.014427956193685532, -0.03592214360833168, -0.01450504269450903, 0.031219888478517532, 0.036538831889629364, -0.003261707490310073, 0.01582835428416729, -0.0042397379875183105, 0.0017794051673263311, -0.03373803570866585, -0.0018388257594779134, -0.021699748933315277, 0.016445044428110123, -0.0005909937899559736, 0.025695381686091423, -0.027057236060500145, 0.009211794473230839, 0.0005006584688089788, -0.0076829190365970135, -0.004907818045467138, -0.03476585075259209, -0.01043232437223196, -0.032864391803741455, 0.0016701997956261039, 0.008948416449129581, -0.0432710237801075, -0.012038285844027996, 0.01596967875957489, 0.005858547054231167, 0.01918160170316696, 0.03253035247325897, 0.004657288081943989, -0.002107021166011691, -0.005823215935379267, 0.022958822548389435, -0.012706366367638111, -0.041318174451589584, 0.018397893756628036, -0.0011241729371249676, -0.011016895063221455, -0.01861630380153656, 0.0031219888478517532, 0.020916040986776352, -0.006012719124555588, -0.019721204414963722, 0.00546669214963913, 0.0056786793284118176, 0.01346438005566597, 0.04016188159584999, 0.0005837669596076012, -0.012635704129934311, -0.0472538061439991, -0.05282970517873764, 0.03170809894800186, -0.00930815190076828, -0.021507034078240395, 0.01478769164532423, 0.033969294279813766, 0.015250208787620068, 0.01061861589550972, 0.012243849225342274, -0.005601593293249607, -0.006073745898902416, 0.022419219836592674, 0.010123980231583118, 0.008819939568638802, -0.014877625741064548, -0.005225798115134239, -0.003118776949122548, 0.021879617124795914, -0.006459176540374756, 0.026067964732646942, -0.004769705235958099, -0.013824115507304668, -0.00475364550948143, -0.00422046659514308, -0.010779212228953838, -0.011929080821573734, 0.004053446464240551, -0.0021407464519143105, -0.03931393474340439, 0.02208518050611019, -0.03725830465555191, 0.029652470722794533, -0.009558681398630142, -0.004798612557351589, 0.028470482677221298, 0.005848911125212908, 0.010779212228953838, -0.0166506078094244, -0.023588359355926514, -0.010323119349777699, 0.004371426533907652, 0.01504464540630579, 0.013695638626813889, -0.005119804758578539, -0.01300828717648983, 0.027005845680832863, -0.007265368942171335, -0.0023832465521991253, -0.01319457869976759, -0.05406308174133301, -0.014607824385166168, -0.016188090667128563, -0.013400141149759293, 0.03214492276310921, -0.022136570885777473, -0.0296010784804821, -0.009359542280435562, 0.006427057087421417, 0.0021873193327337503, -0.03204214200377464, 0.011177490465342999, 0.02687736786901951, -0.008312455378472805, -0.002717286581173539, -0.02257339283823967, -0.1605447381734848, 0.012963320128619671, 0.008678615093231201, -0.015006102621555328, 0.031579624861478806, -0.005864970851689577, 0.014453652314841747, 0.029780946671962738, -0.005887454375624657, -0.010335966944694519, -0.0008511595078743994, 0.007747157476842403, -0.006272885017096996, -0.023125842213630676, -0.018487825989723206, 0.011132524348795414, -0.021841073408722878, 0.013836963102221489, 0.044530097395181656, 0.021763987839221954, 0.03175949305295944, -0.034508898854255676, 0.01641934923827648, -0.0038896382320672274, 0.017228752374649048, 0.01007901318371296, 0.01411961205303669, 0.0052675530314445496, -0.024012334644794464, -0.03173379600048065, -0.012616432271897793, -0.0006488083745352924, 0.00912186037749052, -0.021738292649388313, 0.04576347395777702, -0.006616560742259026, 0.004159439820796251, 0.012687094509601593, -0.012494378723204136, 0.006822123657912016, 0.016496434807777405, 0.0055694738402962685, 0.024153659120202065, 0.008376694284379482, -0.014312326908111572, 0.003671227488666773, 0.023215776309370995, -0.012603584676980972, 0.010387358255684376, 0.015352990478277206, -0.005691526923328638, -0.01282199565321207, 0.04812745004892349, -0.014222393743693829, 0.009455900639295578, 0.0025438426528126, 0.0021519882138818502, 0.019862528890371323, -0.0005689117824658751, 0.022098027169704437, -0.025374189019203186, -0.023729683831334114, -0.007374574430286884, 0.012320935726165771, 0.018115242943167686, -0.01345153246074915, -0.0007086304249241948, -0.015417228452861309, -0.04481274634599686, 0.001403610222041607, -0.0050941091030836105, -0.018179481849074364, 0.008774972520768642, -0.005771825090050697, -0.013245969079434872, -0.015340142883360386, -0.013297359459102154, -0.00233827973715961, 0.001989786047488451, 0.020594848319888115, -0.0010511017171666026, 0.007451660465449095, -0.03833751007914543, -0.020941736176609993, -0.007111196871846914, 3.0889663321431726e-05, -0.00497526815161109, -0.025541208684444427, 0.003578081727027893, -0.00226279953494668, 0.019001733511686325, -0.02402518130838871, -0.020055245608091354, -0.0324532650411129, 0.024950215592980385, 0.01450504269450903, 0.002857005223631859, 0.008248217403888702, 0.007143315859138966, -0.014839082956314087, 0.007625104393810034, -0.006012719124555588, -0.0197854433208704, 0.006603713147342205, 0.027391277253627777, 0.005492387805134058, 0.003844671417027712, 0.0195027943700552, 0.031040020287036896, 0.0029212436638772488, -0.007702190428972244, 0.007329607382416725, 0.024333525449037552, 0.014736301265656948, -0.03111710585653782, 0.00416907574981451, 0.0049142418429255486, -0.014890473335981369, 0.0014935440849512815, -0.005527718923985958, 0.04907817766070366, -0.009462323971092701, 0.01576411724090576, -0.016072461381554604, -0.021789683029055595, -0.004955996759235859, -0.11234020441770554, -0.01437656581401825, 0.008498746901750565, -0.0016139911022037268, 0.023781076073646545, 0.009006231091916561, -0.012481531128287315, -0.005412089638411999, -0.009526562876999378, 0.00883921142667532, -0.029986510053277016, -0.017061732709407806, 0.003097899490967393, -0.0008254641434177756, 0.00237361085601151, 0.001415654900483787, 0.024551937356591225, -0.02379392273724079, -0.018860409036278725, 0.013824115507304668, -0.00866576749831438, 0.013682790100574493, 0.0018420376582071185, 0.0005938041722401977, -0.01034881453961134, -0.022650478407740593, -0.01990107260644436, 0.004075929988175631, 0.008222522214055061, 0.01605961285531521, -0.012886233627796173, -0.01477484405040741, 0.004798612557351589, 0.0014309115940704942, -0.00501059927046299, -0.00844093319028616, -0.01518596988171339, 0.0058424873277544975, 0.018179481849074364, -0.03481724113225937, 0.005042718723416328, 0.008273912593722343, -0.014222393743693829, -0.018577760085463524, 0.005335003603249788, 0.021134451031684875, -0.018950343132019043, 0.005235434044152498, -0.020967431366443634, -0.0005661013419739902, -0.015866898000240326, -0.012487955391407013, -0.010509410873055458, -0.0029196375980973244, 0.022984517738223076, -0.008402389474213123, 0.005344639532268047, 0.0038221878930926323, 0.0023109782487154007, -0.0064977193251252174, -0.007291064597666264, -0.0100340461358428, -0.004901393782347441, 0.00235594529658556, 0.02046637050807476, 0.003934605047106743, 0.025271408259868622, -0.007053382229059935, 0.021738292649388313, -0.013836963102221489, -0.004766493104398251, 0.009937688708305359, -0.007406693417578936, -0.01503179781138897, -0.011941928416490555, 0.010894841514527798, -0.03006359562277794, -1.917116424010601e-05, 0.01688186638057232, -0.02360120788216591, -0.01463351957499981, -0.015481467358767986, 0.00886490661650896, -0.04165221378207207, 0.008774972520768642, -0.007220402359962463, 0.013785571791231632, 0.01810239627957344, 0.020312199369072914, 0.0018468555063009262, 0.006372454576194286, 0.0257596205919981, 0.005097321234643459, -0.007393845822662115, -0.00852444302290678, 0.003944240976125002, 0.003921757452189922, -0.003542750608175993, 0.034174855798482895, 0.012963320128619671, -0.0034592407755553722, -0.005129440687596798, -0.06665381789207458, 0.020247960463166237, -0.005312520079314709, -0.024205049499869347, 0.009006231091916561, -0.021340014412999153, 0.01810239627957344, -0.007920601405203342, -0.00983490701764822, -0.0012470289366319776, -0.03399498760700226, 0.016522129997611046, 0.015083189122378826, -0.009687158279120922, -0.03430333361029625, -0.027905184775590897, 0.010168947279453278, 0.014543585479259491, 0.013798419386148453, 0.013033982366323471, -0.0037740091793239117, 0.01770411804318428, 0.00035451597068458796, 0.0024105480406433344, -0.01910451613366604, 0.01580265909433365, -0.018834713846445084, 0.02058199979364872, -0.008004111237823963, -0.005877818446606398, 0.0010077407350763679, -0.030526112765073776, -0.014569281600415707, 0.017832593992352486, -0.0016356715932488441, 0.0036744396202266216, 0.01516027469187975, -0.006398149766027927, 0.03250465914607048, 0.04488983005285263, -0.041857775300741196, -0.021481338888406754, 0.028701741248369217, -0.009828483685851097, 0.020170874893665314, 0.008350999094545841, -0.026119355112314224, 0.01372133381664753, 0.029703861102461815, -0.0024635447189211845, 0.002767071360722184, 0.022444915026426315, -0.004085565451532602, -0.018693389371037483, 0.015404380857944489, -0.01904027722775936, 0.006182950921356678, 0.003391790436580777, 0.001872550928965211, -0.04306545853614807, 0.02089034579694271, -0.008858482353389263, 0.008633648045361042, -0.008402389474213123, 0.03065459057688713, -0.020363589748740196, 0.011813451535999775, -0.009738549590110779, 0.016290871426463127, -0.03810625150799751, -0.011408749036490917, 0.010316696017980576, 0.012269544415175915, 0.03283869847655296, -0.011466563679277897, 0.03453459218144417, -0.005691526923328638, -0.0018420376582071185, -0.016162395477294922, 0.015070341527462006, -0.01384981069713831, 0.00801053550094366, -0.01622663252055645, 0.023678293451666832, 0.017434315755963326, -0.009276032447814941, 0.008961264044046402, -0.0008688250673003495, 0.007528746500611305, 0.033095650374889374, -0.01437656581401825, 0.020916040986776352, -0.011877690441906452, -0.0022611934691667557, 0.011209609918296337, 0.0032103166449815035, -8.142726073856466e-06, 0.006536262575536966, 0.0005725251976400614, 0.00130082864779979, 0.006234341766685247, 0.0092503372579813, 0.00238003465346992, -0.02419220097362995, -0.012661399319767952, 0.0028682469855993986, -0.007779276464134455, -0.02114729955792427, 0.02081325836479664, 0.01704888604581356, 0.007034110836684704, -0.012571465224027634, 0.006950600538402796, 0.01490332093089819, -0.015584249049425125, 0.0007740733562968671, -0.02530995011329651, -0.0193614698946476, -0.007233249954879284, 0.026337765157222748, 0.012308087199926376, 0.012526498176157475, 0.05545063316822052, -0.011273848824203014, 0.008832787163555622, 0.03296717256307602, 0.0026883792597800493, -0.0035620222333818674, 0.0037964924704283476, -0.003378942608833313, 0.01467206235975027, -0.010695702396333218, -0.012237424962222576, -0.007689342834055424, 0.012963320128619671, -0.016920408234000206, -0.007862786762416363, 0.005704374518245459, -0.007310335990041494, 0.08155713975429535, 0.0050780498422682285, -0.018295111134648323, 0.011672127060592175, 0.0013690820196643472, 0.0091090127825737, 0.00899338349699974, -0.002522965194657445, -0.011974047869443893, 0.0027702832594513893, 0.009956960566341877, -0.013284511864185333, 0.026337765157222748, -0.024063725024461746, -0.040033403784036636, 0.006022355053573847, 0.006375666242092848, 0.02248345874249935, -0.032350484281778336, -0.001084023853763938, 0.03797777369618416, -0.006327487528324127, 0.011627160012722015, -0.000627127883490175, 0.018860409036278725, -0.007663647644221783, 0.02587524801492691, 0.0022338922135531902, -0.0019817561842501163, -0.00845378078520298, -0.005810367874801159, -0.007194706704467535, -0.014081069268286228, -0.013798419386148453, 0.018179481849074364, -0.03569088503718376, 0.008151859976351261, -0.0009443052695132792, 0.027134323492646217, 0.010201066732406616, -0.002991905901581049, 0.022676173597574234, -0.010798484086990356, -0.028316309675574303, 0.033378299325704575, 0.002974240342155099, -0.003545962506905198, 0.0012855720706284046, 0.002103809267282486]\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"chunk_embedding\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 233,\n        \"samples\": [\n          \"[-0.00567989144474268, -0.0028591237496584654, 0.009096776135265827, -0.015201780013740063, -0.015879403799772263, 0.03454601392149925, 0.008041984401643276, -0.02856246940791607, -0.013578040525317192, 0.013117768801748753, 0.013143339194357395, -0.002135153394192457, -0.019037384539842606, -0.01711958274245262, -0.010330562479794025, -0.008655681274831295, 0.032781634479761124, -0.010151566937565804, 0.021850161254405975, -0.027258364483714104, 0.005321901757270098, -0.009908645413815975, -0.021338747814297676, -0.0069935861974954605, -0.0036278427578508854, -0.014971643686294556, 0.019024599343538284, -0.016557026654481888, 0.025903116911649704, -0.02869032323360443, 0.021760664880275726, -0.012529642321169376, 0.006654774304479361, -0.026619097217917442, -0.03012228198349476, -0.005008660722523928, 0.010899510234594345, -0.004653867334127426, 0.0052835457026958466, -3.338674287078902e-05, 0.00540181016549468, 0.0015374382492154837, -0.00792052410542965, -0.01104654185473919, -0.027053799480199814, 0.01573876477777958, -0.017336932942271233, -0.01049037929624319, -0.035926830023527145, 0.024432802572846413, 0.0049894824624061584, 0.014217308722436428, 0.007102261297404766, -0.017362505197525024, -0.02326933480799198, 0.011442887596786022, 0.012216401286423206, 0.011212751269340515, -0.022502215579152107, 0.002552275313064456, 0.01356525532901287, -0.014051099307835102, -0.019369803369045258, -0.009787185117602348, -0.0019705418962985277, 0.020584411919116974, -0.013488543219864368, -0.011545170098543167, 0.009020064026117325, 0.011097683571279049, 0.01566205359995365, 0.01230589859187603, 0.00596756162121892, -0.02222093753516674, 0.056255538016557693, -0.025187138468027115, 0.0023397188633680344, 0.003022136865183711, -0.016147896647453308, -0.021620025858283043, 0.02254057116806507, -0.04347018897533417, -0.036975231021642685, -0.007204544264823198, 0.031656526029109955, 0.00786938238888979, -0.0030029588378965855, 0.03377889096736908, -0.02461179718375206, -0.003602272132411599, -0.010829190723598003, 0.019663866609334946, 0.01898624375462532, -0.0033273871522396803, 0.001807528780773282, -0.009301341138780117, -0.005350668448954821, 0.0316820964217186, 0.016147896647453308, -0.016659310087561607, -0.017273006960749626, -0.003563916077837348, -0.029278449714183807, -0.004625100176781416, 0.0037525000516325235, -0.001191434683278203, -0.008265728130936623, 0.00570865860208869, 0.026516813784837723, -0.007051120046526194, 0.023307692259550095, 0.040401704609394073, 0.008700430393218994, -0.024381661787629128, -0.02692594565451145, 0.002867114497348666, 0.00536984670907259, -0.03554327040910721, 0.0038643719162791967, -0.019472086802124977, 0.01262553222477436, 0.034469302743673325, 0.006357514765113592, -0.00117944844532758, 0.01235704030841589, 0.02940630353987217, -0.0260821133852005, -0.018730536103248596, -0.007843811996281147, -0.008355225436389446, 0.02178623527288437, 0.0014047902077436447, 0.00468902662396431, -0.0316820964217186, -0.017388075590133667, 0.019523227587342262, -0.03380446508526802, 0.022502215579152107, -0.004481264855712652, -0.008374404162168503, 0.014460230246186256, 0.011206358671188354, 0.004008207004517317, -0.007760706823319197, 0.0053123123943805695, 0.00599632877856493, 0.015086712315678596, 0.02273235097527504, -0.013795391656458378, 0.0025330972857773304, 0.011781699024140835, -0.001132302451878786, 0.023090340197086334, -0.0049703046679496765, 0.01142370980232954, -0.013987172394990921, 0.009729650802910328, 0.005424184259027243, -0.03303734213113785, -0.007939701899886131, -0.022757921367883682, 0.02564741112291813, 0.017873918637633324, -0.016224607825279236, -0.0015222555957734585, 0.025289420038461685, -0.008220979943871498, 0.009627368301153183, 0.023333262652158737, 0.008399974554777145, -0.025800835341215134, 0.002760037314146757, -0.041040971875190735, 0.00503103481605649, -0.003659806214272976, 0.008061163127422333, 0.020213637501001358, -0.01691501773893833, -0.007658424321562052, -0.026721380650997162, -0.037077512592077255, 0.00793330930173397, -0.006731486413627863, 0.00829769205302, 0.0036693953443318605, 0.020494915544986725, -0.0018251086585223675, -0.003554327180609107, 0.0026449691504240036, 0.004158434923738241, 0.01940816082060337, 0.009697687812149525, -0.00855339877307415, -0.016314106062054634, -0.6296528577804565, -0.02973872236907482, -0.03188665956258774, -0.022182580083608627, 0.017976200208067894, 0.02452230080962181, 0.011596311815083027, 0.02638895995914936, 0.006814591120928526, -0.04602725803852081, -0.01679994910955429, 0.016505885869264603, -0.011487636715173721, -0.015048355795443058, -0.005603179335594177, -0.005596786737442017, 0.013488543219864368, 0.02305198460817337, 0.03147752955555916, 0.009768006391823292, -0.01387210376560688, 0.018436472862958908, 0.0009045634651556611, 0.0014303609495982528, 0.027974344789981842, 0.020635554566979408, 0.013999957591295242, -0.0247268658131361, -0.019855648279190063, -0.011123253963887691, -0.033420901745557785, 0.03953230008482933, -0.007773492485284805, 0.016966158524155617, 0.05339161679148674, -0.023333262652158737, -0.020520485937595367, 0.03845833241939545, -0.00758810481056571, 0.04722908139228821, 0.010030106641352177, 0.0039314948953688145, 0.01802734285593033, -0.012702244333922863, 0.020047428086400032, 0.01038170326501131, 0.03109396994113922, 0.002397252945229411, -0.008642896078526974, -0.038637325167655945, 0.002312550088390708, -0.02223372273147106, -0.007735136430710554, -0.0069104814901947975, -0.0021015917882323265, -0.006578062195330858, 0.02378075011074543, -0.019062954932451248, -0.006168930791318417, -0.018615469336509705, -0.005913224071264267, 0.007051120046526194, 0.0059355986304581165, 0.005823726765811443, 0.003650217317044735, 0.025992615148425102, -0.030377989634871483, -0.0033753321040421724, 0.0157515499740839, -0.025736907497048378, 0.015099497511982918, 0.024471158161759377, -0.01802734285593033, 0.02012413926422596, -0.01167302392423153, -0.005232403986155987, 0.03784463554620743, -0.006872125435620546, 0.036770664155483246, -0.019996285438537598, 0.0043182517401874065, 0.0020057016517966986, -0.013156124390661716, 0.013283978216350079, 0.02023920789361, -0.00267213792540133, -0.025378918275237083, -0.03117068111896515, 0.001745200133882463, 0.015073927119374275, 0.0007491415599361062, 0.012702244333922863, -0.0024739650543779135, 0.0016213421477004886, -0.03531313315033913, 0.0252382792532444, -0.011455672793090343, 0.0025107229594141245, -0.015406345948576927, -0.036233678460121155, 0.022553356364369392, -0.018743321299552917, 0.013411831110715866, 0.009154310449957848, 0.016774378716945648, -0.00534107955172658, -0.006533313542604446, 0.013322333805263042, 0.016211822628974915, -0.04298434406518936, 0.02043098770081997, -0.0065173315815627575, -0.01594332978129387, -0.018206337466835976, -0.00378126697614789, -0.02199080027639866, 0.029483014717698097, 0.003551130648702383, -0.006002721376717091, -0.026951516047120094, 0.013104982674121857, 0.015022785402834415, 0.008764357306063175, -0.02263006754219532, 0.01949765719473362, 0.004059348255395889, -0.013705894351005554, -0.021108612418174744, -0.009416409768164158, 0.01235704030841589, 0.023844676092267036, -0.0074730366468429565, 0.022169794887304306, -0.017694924026727676, 0.03329304978251457, -0.011391745880246162, 0.03574783727526665, 0.009013671427965164, -0.0005497700185514987, -0.018385332077741623, -0.022655639797449112, 0.0011043344857171178, 0.027514072135090828, 0.0014719133032485843, -0.006440619472414255, -0.019523227587342262, -0.0007663219003006816, -0.003095652675256133, -0.05569298192858696, 0.0004874414298683405, -0.001994514372199774, -0.012203616090118885, -0.02491864562034607, -0.0007183768320828676, -0.003966654650866985, -0.011762521229684353, -0.0158410482108593, 0.0030253331642597914, -0.014217308722436428, -0.011225536465644836, 0.005497700069099665, 0.019638296216726303, -0.0037461072206497192, 0.0030541003216058016, -0.012868453748524189, -0.012651103548705578, 0.01793784461915493, 0.0034648296423256397, 0.0019817291758954525, -0.017017299309372902, 0.02347390167415142, -0.046257395297288895, -0.008764357306063175, -0.010522342287003994, -0.00010278222180204466, 0.01930587738752365, 0.013437402434647083, -0.024560656398534775, -0.012094940058887005, -0.009704080410301685, 0.004292680881917477, -0.018321406096220016, -0.029585298150777817, -0.004158434923738241, 0.046359676867723465, 0.00974882859736681, 0.002310951938852668, 0.015521413646638393, -0.028715893626213074, 0.04139896109700203, 0.0016684880247339606, -0.00855339877307415, 0.009633760899305344, -0.008112303912639618, 0.009135131724178791, 0.021108612418174744, 0.021978015080094337, 0.016531456261873245, -0.0009197460603900254, 0.019740579649806023, -0.003918709699064493, 0.016544241458177567, 0.02585197612643242, 0.004203183576464653, 0.018973458558321, -0.038432758301496506, -0.0012329871533438563, -0.024036455899477005, 0.019344232976436615, 0.011462065391242504, -0.012638317421078682, -0.043444618582725525, 0.001614949433133006, -0.009410017170011997, 0.002686521504074335, 0.021632811054587364, 0.011589919216930866, 0.010138781741261482, -0.036463815718889236, 0.008201801218092442, 0.02700265683233738, 0.018104054033756256, 0.0036214501596987247, -0.013897675089538097, -0.030889403074979782, -0.028920460492372513, 0.021006328985095024, 0.02085290476679802, 0.028383474797010422, -0.0021687147673219442, -0.013526899740099907, 0.004129667766392231, -0.01142370980232954, 0.021824590861797333, -0.0017755653243511915, -0.02481636218726635, 0.014447445049881935, -0.02181180566549301, 0.04464643821120262, -0.01563648134469986, -0.01594332978129387, 0.019357018172740936, 0.01392324548214674, -0.007946094498038292, 0.008745178580284119, -0.02095518819987774, -0.0023636913392692804, 0.012606354430317879, -0.02076340653002262, 0.012862061150372028, 0.01194790843874216, 0.010618232190608978, -0.011442887596786022, -0.0088091054931283, 0.010861154645681381, -0.015419131144881248, 0.013066627085208893, 0.018104054033756256, -0.01604561321437359, 0.012817312963306904, -0.014715936966240406, 0.0030013606883585453, 0.008764357306063175, -0.008054769597947598, 0.0021223679650574923, 0.0035958795342594385, 0.020929615944623947, 0.008598146960139275, -0.0012673477176576853, -0.026746951043605804, 0.009825540706515312, -0.003397706663236022, 0.028741465881466866, 0.003960262052714825, 0.0055328598245978355, 0.005900438874959946, 0.0007982852403074503, -0.0035703086759895086, 0.03526199236512184, -0.0030988489743322134, -0.019689437001943588, -0.030480273067951202, 0.027283934876322746, 0.036540526896715164, -0.021875733509659767, -0.009320519864559174, -0.0037013585679233074, 0.020072998479008675, -0.0423450767993927, 0.005481718573719263, 0.0006928060902282596, 0.0002503131690900773, 0.007236507721245289, -0.0020872082095593214, -0.0006932056276127696, -0.021223679184913635, 0.024547871202230453, -0.010586269199848175, -0.017643781378865242, -0.013079412281513214, -0.01566205359995365, -0.009710473008453846, -0.005050213076174259, -0.03605468571186066, 0.0026817270554602146, 0.019433731213212013, -0.020188067108392715, -0.012746993452310562, 0.0233716182410717, -0.01594332978129387, -0.008834675885736942, -0.007178973406553268, -0.007543356157839298, 0.008093126118183136, -0.018525971099734306, 0.009646546095609665, -0.010413667187094688, 0.010343347676098347, 0.03002000041306019, -0.006641989108175039, -0.030326848849654198, -0.01910131238400936, -0.016199037432670593, -0.01938258856534958, 0.04807291179895401, 0.061676524579524994, -0.020469343289732933, 0.023090340197086334, 0.01543191634118557, 0.003384921234101057, -0.01443465892225504, -0.015099497511982918, 0.02605654112994671, -0.006213679909706116, 0.0022598104551434517, -0.001626136596314609, 0.005436969920992851, 0.01543191634118557, 0.004522817209362984, -0.006236054003238678, 0.02003464289009571, -0.03014785423874855, 0.030582554638385773, -0.02251500077545643, -0.0325259268283844, -0.0006500551244243979, -0.015393560752272606, 0.0054593440145254135, -0.02784649096429348, -0.017771635204553604, 0.0013057037722319365, 0.015764335170388222, 0.021568885073065758, -0.029687581583857536, -0.00029805846861563623, -0.008412759751081467, 0.0033177980221807957, -0.004541995469480753, 0.0007998833898454905, -0.01928030699491501, -0.011839233338832855, 0.016326891258358955, 0.005833315663039684, -0.014997215010225773, 0.03055698424577713, -0.0024372071493417025, -0.004778524395078421, -0.001975336577743292, 0.020098568871617317, -0.0035958795342594385, 0.010298598557710648, 0.03219350799918175, -0.001866661012172699, -0.016710450872778893, 0.0313241071999073, 0.01199265755712986, -0.021146968007087708, 0.00923741515725851, 0.0064342268742620945, 0.021325962617993355, 0.03147752955555916, -0.023934174329042435, -0.0014854978071525693, 0.04935144633054733, -0.0010092435404658318, -0.0034552407450973988, -0.0034264735877513885, 0.0029054705519229174, -0.0008941753767430782, -0.025724122300744057, -0.004490853752940893, -0.01981729082763195, -0.022195367142558098, 0.0094867292791605, -0.0036630025133490562, 0.007767099887132645, -0.004947930108755827, 0.009978964924812317, 0.034162454307079315, 0.012638317421078682, 0.041347820311784744, -0.03347204253077507, 0.008578969165682793, -0.00298697710968554, -0.004526013508439064, -0.010861154645681381, 0.017068441957235336, 0.00975522119551897, -0.002544284565374255, 0.013744250871241093, 0.007747921627014875, 0.002037665108218789, -0.06050027161836624, 0.030071141198277473, -0.011609097011387348, 0.016978943720459938, 0.013386260718107224, -0.009461158886551857, -0.013450187630951405, 0.024279378354549408, 0.003599075833335519, 0.02283463440835476, -0.01111046876758337, -0.008067555725574493, -0.0008222577744163573, -0.03664281219244003, -0.0006664363318122923, -0.006475779227912426, 0.012708636932075024, -0.019957929849624634, -0.022604497149586678, 0.0018203140934929252, -0.017375290393829346, 0.00943558756262064, 0.003169168485328555, -0.01204379927366972, 0.01267667394131422, 0.017285792157053947, 0.012631924822926521, 0.021709522232413292, -0.001994514372199774, 0.0367450937628746, -0.006146556697785854, -0.015419131144881248, 0.00696162274107337, -0.022668424993753433, 0.027079369872808456, 0.005603179335594177, -0.03127296268939972, 0.006162538193166256, -0.014332376420497894, -0.021939659491181374, -0.003333779750391841, 0.012152474373579025, 0.005139710381627083, -0.009768006391823292, -0.011346997693181038, -0.009103168733417988, -0.010605446994304657, -0.019050169736146927, -0.003503185696899891, 0.012644710019230843, -0.019689437001943588, 0.006383085623383522, -0.0008214586996473372, -0.011749736033380032, -0.0012377816019579768, 0.010867547243833542, -0.029355162754654884, -0.030531413853168488, -0.0008018811349757016, 0.0021623221691697836, -0.015994472429156303, 0.03856061398983002, 0.012772563844919205, 0.0036182538606226444, 0.003867568215355277, 0.008566183969378471, 0.008304084651172161, -0.01938258856534958, 0.0021623221691697836, -0.01366753876209259, 0.005417791660875082, -0.0048136841505765915, 0.037588927894830704, 0.0009549057576805353, 0.022783491760492325, 0.003982636146247387, -0.011187180876731873, -0.0005853292532265186, -0.008617325685918331, -0.008304084651172161, -0.01485657598823309, 0.030633697286248207, 0.0323469340801239, 0.0350574292242527, 0.027897631749510765, 0.01037531066685915, 0.015674838796257973, 0.023652896285057068, 0.012593569234013557, -0.0023796730674803257, 0.0011219143634662032, -0.01628853566944599, -0.008214587345719337, -0.015419131144881248, -0.006472582928836346, -0.0019465694203972816, -0.007396324537694454, -0.007172580808401108, 0.015355204232037067, -0.0031723647844046354, -0.027693066745996475, -0.020405417308211327, 0.021517742425203323, -0.040401704609394073, 0.01179448515176773, -0.013181694783270359, 0.007345183286815882, -0.006379889324307442, -0.0030029588378965855, -0.0053762393072247505, -0.015380775555968285, 0.007281256373971701, 0.020801763981580734, 0.005849297158420086, 0.006808198522776365, -0.0018394921207800508, -0.0008230568491853774, 0.0023684860207140446, 0.015393560752272606, 0.00028007905348204076, 0.0008294495055451989, -0.013987172394990921, -0.04382817819714546, -0.010547913610935211, -0.006411852780729532, -0.015099497511982918, -0.010982614941895008, 0.006178520154207945, -0.022169794887304306, 0.011078504845499992, -0.005794959608465433, -0.0284601878374815, -0.019574370235204697, -0.0007183768320828676, 0.013424616307020187, 0.0007851003319956362, 0.061676524579524994, 0.01700451411306858, -0.026849232614040375, 0.0015670043649151921, 0.02825562097132206, 0.0031244198326021433, -0.016109539195895195, 0.007134224753826857, 0.010106818750500679, -0.011251107789576054, -0.01740086078643799, 0.007888560183346272, -0.021965229883790016, -0.03232136368751526, -0.014715936966240406, 0.02388303168118, 0.0006644386448897421, -0.00045387991121970117, -0.008016414009034634, -0.005104550626128912, -0.02389581687748432, 0.021568885073065758, -0.010899510234594345, -0.007894952781498432, -0.016326891258358955, -0.029048314318060875, -0.01595611497759819, -0.00103641243185848, 0.00891778152436018, -0.00437578558921814, 0.015086712315678596, -0.025800835341215134, -0.0014039911329746246, -0.011896767653524876, -0.0021591258700937033, 0.000818262342363596, -0.010445630177855492, 0.007051120046526194, -0.02450951375067234, 0.026516813784837723, 0.023716822266578674, 0.009314127266407013, -0.002000907203182578, 0.01690223254263401, -0.005328294355422258, 0.017030084505677223, -0.0049798935651779175, 0.005277153104543686, -0.013232836499810219, -0.009448372758924961, 0.0031499904580414295, 0.021146968007087708, 0.01604561321437359, -0.0119862649589777, -0.014639224857091904, -0.008080340921878815, 0.028102198615670204, 0.023422759026288986, -0.009774399921298027, 0.004238343331962824, -0.01549584325402975, -0.012631924822926521, -0.01350132841616869, -0.02073783613741398, 0.01834697648882866, 0.011231929063796997, -0.07415502518415451, -0.026644667610526085, 0.0002247424708912149, 0.014460230246186256, -0.026158824563026428, 0.005184459034353495, 0.016352461650967598, 0.02825562097132206, -0.004168023820966482, 0.0002792799787130207, -0.03544098883867264, -0.003925102297216654, -0.013309548608958721, 0.014319591224193573, 0.017017299309372902, -0.02554512768983841, 0.03827933594584465, -0.011129646562039852, -0.03045470267534256, -0.017273006960749626, 0.017963415011763573, -0.0012593568535521626, -0.008489471860229969, -0.006213679909706116, -0.014115025289356709, 0.02514878287911415, -0.004439712502062321, -0.009627368301153183, -0.005606375634670258, 0.028025485575199127, 0.0010244260774925351, -0.006478975526988506, 0.00571185490116477, -0.015789905562996864, 0.007952487096190453, 0.0063607110641896725, -0.02179902046918869, 0.015355204232037067, -0.03469943627715111, 0.0002998563868459314, 0.0022070708218961954, 0.018743321299552917, 0.01605839841067791, -0.008994493633508682, -0.0247268658131361, -0.0010899510234594345, -0.013258406892418861, -0.022272078320384026, 0.0074922144412994385, -0.01690223254263401, -0.0242154523730278, -0.0051461029797792435, 0.021645596250891685, 0.008118696510791779, -0.018909530714154243, -0.016927802935242653, 0.007166188210248947, -0.0052100298926234245, -0.04037613421678543, -0.008802712894976139, -0.007147010415792465, 0.02605654112994671, 0.016339676454663277, -0.02638895995914936, -0.03411131352186203, 0.01656981185078621, -0.019625511020421982, 0.017234651371836662, 0.0005222015897743404, -0.00787577498704195, -0.004864825401455164, 0.005849297158420086, -0.010471201501786709, 0.006968015339225531, 0.0070766909047961235, -0.012504071928560734, 0.008227372542023659, 0.007683994714170694, 0.007498607505112886, -0.004957519005984068, 0.02546841651201248, 0.006133771501481533, -0.007006371393799782, 0.002855927450582385, -0.014958858489990234, 0.011391745880246162, 0.010049284435808659, -0.002756841015070677, -0.023691251873970032, -0.012018227949738503, -0.020213637501001358, 0.014664795249700546, 0.024650152772665024, 0.003429669886827469, -0.011346997693181038, -0.0037269294261932373, -0.030838262289762497, 0.0034552407450973988, 0.009307734668254852, -0.01887117512524128, 0.004414141643792391, 0.01564926840364933, 0.004478068556636572, 0.002456385176628828, -0.01824469305574894, 0.001093946397304535, 0.020175281912088394, -0.005424184259027243, 0.017912274226546288, -0.010867547243833542, -0.00019427738152444363, -0.005695872940123081, -0.008598146960139275, 0.002290175762027502, 0.0017148349434137344, 0.008201801218092442, -0.04012042656540871, 0.0012969138333573937, -0.0019353822572156787, -0.027360647916793823, 0.026094898581504822, -0.013347904197871685, -0.010963437147438526, -0.011800877749919891, -0.006801805924624205, -0.026516813784837723, -0.02442001737654209, -0.024535086005926132, 0.03919988125562668, 0.004503639414906502, -0.019050169736146927, 0.016633739694952965, -0.012465715408325195, -0.008674859069287777, 0.008681252598762512, -0.0075561413541436195, 0.02355061285197735, -0.01815519668161869, -0.019254736602306366, 0.016736021265387535, -0.01230589859187603, -0.03198894485831261, 0.003867568215355277, -0.000770317274145782, 0.025391703471541405, -0.003615057561546564, 0.22809062898159027, -0.004580351524055004, -0.0062776063568890095, 0.0065237246453762054, 0.020929615944623947, 0.012222793884575367, 0.031758807599544525, 0.01637803204357624, -0.0017148349434137344, 0.0038004450034350157, -0.012433752417564392, -0.0038867462426424026, -0.005980347283184528, 0.01506114099174738, 0.0005337883485481143, -0.021543312817811966, -0.049709439277648926, 0.0028095804154872894, -0.011327819898724556, -0.04835418984293938, 0.006974407937377691, -0.007690387777984142, 0.00024192278215195984, -0.03462272509932518, -0.002143144141882658, 0.030684838071465492, -0.003242684295400977, -0.008636503480374813, 0.022911345586180687, -0.012919595465064049, -0.0013856121804565191, 0.009966179728507996, -0.024752436205744743, -0.012810920365154743, -0.004247932229191065, -0.0017084423452615738, 0.0028814980760216713, -0.004909574054181576, 0.004110489971935749, 0.0260821133852005, 0.009205451235175133, -0.010816405527293682, -0.0184620451182127, 0.010215493850409985, -0.0053986138664186, -0.016186252236366272, -0.0162629634141922, -0.01804012805223465, 0.004359804093837738, 0.02399810031056404, -0.02324376441538334, -0.012190830893814564, 0.02098075859248638, 0.03876517713069916, 0.0020664320327341557, 0.0004706606559921056, 0.003407295560464263, 0.01648031547665596, -0.007287648972123861, -0.008508649654686451, -0.005123728886246681, 0.005986739881336689, -0.005782174412161112, 0.012484893202781677, -0.002410038374364376, 0.019676651805639267, -0.0265935268253088, 0.002023281529545784, 0.00040633438038639724, -0.036872945725917816, -0.002410038374364376, -0.019433731213212013, 0.006891303230077028, 0.028230050578713417, -0.03953230008482933, -0.020507700741291046, 0.024854719638824463, 0.010106818750500679, -0.020622767508029938, 0.03840718790888786, 0.010982614941895008, -0.0009908645879477262, 0.009135131724178791, 0.0284601878374815, -0.018896745517849922, -0.026619097217917442, 0.021134182810783386, 0.009844718500971794, 0.0006592446006834507, -0.023486686870455742, 0.009032849222421646, 0.01490771770477295, -0.005890849512070417, -0.00855339877307415, -0.005846100859344006, 0.012516857124865055, 0.02723279409110546, 0.02879260666668415, 0.012478500604629517, 0.017784420400857925, -0.04288206249475479, -0.01667209528386593, 0.015598125755786896, 0.016978943720459938, -0.020840119570493698, -0.0009676911286078393, 0.01350132841616869, 0.004915966652333736, 0.02357618324458599, 0.0027999915182590485, -0.019945144653320312, -0.011960694566369057, 0.032602641731500626, 0.007907738909125328, 0.010650196112692356, -0.008003628812730312, 0.003145196009427309, -0.01163466740399599, 0.010304991155862808, -0.011168002150952816, 0.043649181723594666, -0.025711337104439735, 0.002442001597955823, 0.014946073293685913, -0.009640153497457504, -0.00980636291205883, -0.003989028744399548, 0.003832408459857106, 0.0017499946989119053, -0.038867462426424026, 0.01731136254966259, -0.022489428520202637, 0.026094898581504822, -0.005171673838049173, -0.01460086926817894, 0.016237393021583557, 0.0021894911769777536, 0.0053986138664186, 0.0008989698835648596, -0.015112282708287239, 0.009831933304667473, -0.01105932705104351, -0.002274194033816457, 0.017042869701981544, -0.01272781565785408, -0.06142081692814827, 0.007862989790737629, 0.0026593527290970087, -0.0027999915182590485, -0.015035570599138737, -0.025685766711831093, -0.040427275002002716, -0.00881549809128046, -0.018398117274045944, 0.03065926767885685, 0.007115046959370375, 0.0042543248273432255, 0.001095544663257897, 0.012024620547890663, -0.006076237186789513, -0.015508628450334072, 0.001829903107136488, 0.025404488667845726, -0.00599952507764101, -0.006530117243528366, 0.012197223491966724, -0.15997028350830078, 0.028178909793496132, 0.010822798125445843, -0.01983007602393627, 0.03431587666273117, 0.007645638659596443, 0.011513207107782364, 0.020251993089914322, -0.03334419056773186, -0.0008853854378685355, 0.00943558756262064, -0.0018538756994530559, -0.011295855976641178, -0.024880290031433105, -0.029559727758169174, -0.0062680174596607685, -0.03702637180685997, -0.003928298596292734, 0.02606932632625103, 0.003960262052714825, 0.046359676867723465, -0.02417709492146969, 0.013641967438161373, 0.007294041570276022, 0.00659723998978734, 0.012382610701024532, -0.013514114543795586, 0.011193573474884033, -0.020942403003573418, -0.021939659491181374, -0.039353303611278534, -0.0077223507687449455, -0.02525106444954872, -0.005983543582260609, 0.012158866971731186, -0.00599632877856493, 0.012715029530227184, -0.0066803451627492905, -0.015521413646638393, 0.02513599582016468, -0.002239034278318286, 0.007466644048690796, 0.039890289306640625, 0.007786277681589127, 0.0001282530283788219, 0.019152453169226646, 0.019523227587342262, 0.0019481675699353218, 0.01573876477777958, -0.002795197069644928, 0.015789905562996864, -0.02284741960465908, 0.04211493954062462, -0.0022214544005692005, 0.004212772473692894, 0.011775306425988674, 0.024547871202230453, 0.019855648279190063, 0.014345161616802216, -0.006865732371807098, -0.0126191396266222, 0.0037461072206497192, 0.022898560389876366, -0.013424616307020187, -0.001372826867736876, 0.000951709458604455, 0.005814137402921915, -0.030812691897153854, -0.021146968007087708, -0.004570762161165476, 0.011244714260101318, -0.026542384177446365, 0.006942444480955601, -0.03160538524389267, 0.0107972277328372, 0.021658381447196007, 0.006213679909706116, -0.009410017170011997, -0.004832861945033073, 0.0257496926933527, -0.006373496726155281, 0.02961086854338646, -0.02889489009976387, -0.001746798399835825, 0.02011135406792164, 0.01361639704555273, -0.008368010632693768, -0.009947001934051514, 0.016812734305858612, -0.018947888165712357, -0.007549748755991459, -0.019331447780132294, -0.012229186482727528, -0.03771677985787392, 0.023959744721651077, 0.009512299671769142, -0.00204565585590899, 0.011251107789576054, 0.0215944554656744, -0.04101540148258209, -0.0009101570467464626, -0.007683994714170694, 0.0019369804067537189, 0.019459301605820656, 0.01980450563132763, 0.02023920789361, -0.012529642321169376, 0.01983007602393627, 0.0069296592846512794, 0.007690387777984142, -0.015278492122888565, 0.0011107272002846003, 0.017605425789952278, 0.019868433475494385, -0.018641039729118347, 0.010285813361406326, -0.015406345948576927, -0.03444373235106468, 0.003966654650866985, 0.0007711163489148021, 0.06474500894546509, -0.017452001571655273, 0.008770749904215336, 0.007978057488799095, -0.02605654112994671, 0.005523270927369595, -0.0721605122089386, -0.0012689458671957254, -0.005337883252650499, -0.02034149132668972, 0.03408573940396309, 0.04641081765294075, 0.010707730427384377, -0.010944259352982044, -7.895952148828655e-05, 0.014984428882598877, -0.01517620962113142, -0.01554698497056961, 0.0010755675612017512, 0.0034840076696127653, -0.00039095201645977795, -0.004164827521890402, 0.01866661012172699, -0.027974344789981842, -0.03531313315033913, 0.00937805324792862, -0.022348791360855103, 0.01042645238339901, 0.0060187033377587795, -0.00633514067158103, -0.002660950878635049, -0.011174395680427551, -0.029866576194763184, 0.010343347676098347, 0.014217308722436428, 0.01480543427169323, 0.001984925474971533, -0.021031899377703667, -0.009390839375555515, -0.02054605633020401, -0.009147917851805687, 0.005478522274643183, -0.029252879321575165, 0.015304063446819782, 0.003989028744399548, -0.03117068111896515, 0.017388075590133667, 0.0013808177318423986, -0.01043284498155117, -0.014588083140552044, 0.01679994910955429, 0.018206337466835976, -0.01679994910955429, 0.013309548608958721, -0.00406574085354805, -0.007549748755991459, 0.001432758173905313, 0.008713215589523315, -0.0035095782950520515, -0.01548305805772543, 0.02096797339618206, -0.004928752314299345, -0.007524177897721529, -0.0045004431158304214, 0.0008350431453436613, -0.025519557297229767, -0.0005553636001423001, -0.008949744515120983, -0.010042891837656498, -0.005609571933746338, 0.0029310414101928473, -2.9765891667921096e-05, -0.00468902662396431, -0.02223372273147106, 0.01636524684727192, -0.005264367442578077, 1.2691706615441944e-05, 0.018078483641147614, 0.002745653735473752, -0.01815519668161869, -0.027258364483714104, -0.013335119001567364, -0.0342135950922966, 0.01099540013819933, -0.007741529028862715, -0.032909490168094635, 0.006303177215158939, -0.018909530714154243, 0.004570762161165476, 0.0016317302361130714, 0.002617800375446677, 0.015316848643124104, -0.009390839375555515, -0.012740600854158401, 0.00730043463408947, -0.004778524395078421, 0.0067634498700499535, 0.01865382492542267, 0.01804012805223465, -0.009768006391823292, -0.0036566099151968956, 0.0215944554656744, -0.0023700841702520847, -0.015406345948576927, 0.04321448132395744, 0.029687581583857536, 0.013769821263849735, -0.003950673155486584, -0.07707008719444275, 0.022489428520202637, -0.008853854611515999, 0.0023461116943508387, 0.0024276182521134615, -0.015815477818250656, 0.03549212962388992, -0.0020344688091427088, 0.01719629392027855, 0.006948837544769049, -0.020597197115421295, 0.022489428520202637, -0.006450208835303783, 0.0003324190911371261, -0.029687581583857536, -0.008994493633508682, 0.01231229119002819, 0.015278492122888565, 0.02254057116806507, 0.015444701537489891, -0.01423009391874075, 0.007115046959370375, 0.0021479385904967785, 0.02973872236907482, -0.02367846667766571, 0.02211865410208702, -0.02418988011777401, 0.02576247788965702, -0.01549584325402975, 0.0069871935993433, 0.015457487665116787, -0.03127296268939972, -0.013936030678451061, 0.022655639797449112, -0.010362525470554829, 0.013910460285842419, 0.03214236721396446, -0.013641967438161373, 0.02117253839969635, 0.020085783675312996, -0.02984100580215454, -2.9247735255921725e-06, 0.030071141198277473, -0.00861093308776617, 0.007505000103265047, 0.016122326254844666, -0.016147896647453308, 0.007191759068518877, 0.056460101157426834, -0.006872125435620546, 0.0028351512737572193, 0.021760664880275726, 0.0064981537871062756, -0.04267749562859535, 0.027053799480199814, -0.009020064026117325, 0.02410038374364376, -0.0047241863794624805, 0.032704923301935196, -0.02138988859951496, 0.02899717167019844, -0.011883982457220554, 0.010854761116206646, -0.013846533372998238, 0.02617160975933075, -0.006156145595014095, 0.03283277526497841, -0.016109539195895195, 0.02649124339222908, -0.019561585038900375, -0.014971643686294556, -0.011756128631532192, 0.01429402083158493, 0.018641039729118347, 0.013884888961911201, 0.014792649075388908, -0.003445651615038514, -0.012996307574212551, -0.027488501742482185, 0.01667209528386593, 0.006942444480955601, 0.011212751269340515, -0.036131396889686584, -0.007862989790737629, 0.02022642269730568, -0.015879403799772263, -0.004973500967025757, 0.011788091622292995, -0.004465283360332251, 0.01993235945701599, -0.0033177980221807957, -0.0004442908975761384, -0.009058419615030289, -0.004209576174616814, -0.005775781348347664, -0.013629182241857052, -0.0029933699406683445, -0.0017803598893806338, 0.03526199236512184, 0.019229166209697723, 0.004832861945033073, -0.014536942355334759, 0.016416387632489204, -0.023640111088752747, -0.029662011191248894, 0.012472108006477356, -0.019126882776618004, -0.018896745517849922, 0.007888560183346272, 0.004036974161863327, 0.031221823766827583, -0.0018634647130966187, -0.0053858282044529915, 0.013424616307020187, -0.04116882383823395, -0.004810487851500511, -0.0014127810718491673, -0.010643803514540195, -0.026312248781323433, 0.03574783727526665, 0.009883075021207333, 0.0072429003193974495, 0.03449487313628197, -0.010145174339413643, 0.00812508910894394, 0.0126638887450099, -0.008885817602276802, -0.015687623992562294, 0.016147896647453308, 0.006616418249905109, 0.01563648134469986, -0.006427834276109934, 0.008789927698671818, 0.004081722814589739, -0.002395654795691371, 0.007709565572440624, 0.01262553222477436, 0.009045634418725967, -0.005452951416373253, 0.0826444998383522, 0.006571669597178698, -0.01865382492542267, 0.001373625942505896, 0.00923741515725851, 0.023742394521832466, 0.02450951375067234, -0.01288123894482851, -0.026721380650997162, 0.019791720435023308, 0.018845604732632637, 0.0007239704136736691, 0.029636438935995102, -0.021415460854768753, -0.03296063095331192, 0.0036789842415601015, -0.009307734668254852, 0.0054593440145254135, -0.02118532359600067, -0.003525560023263097, 0.024125954136252403, -0.007741529028862715, 0.014383518137037754, -0.011589919216930866, 0.018641039729118347, -0.006482172291725874, 0.01595611497759819, 0.02085290476679802, 0.0052292076870799065, 0.0014838996576145291, 0.02054605633020401, -0.00727486377581954, -0.014958858489990234, -0.015316848643124104, 0.02085290476679802, -0.0538518913090229, -0.001890633488073945, 0.003541541751474142, 0.009448372758924961, 0.010234671644866467, -0.020725050941109657, 0.031145110726356506, -0.02378075011074543, -0.020277563482522964, 0.004529209807515144, -0.013514114543795586, 0.018308620899915695, -0.017567070201039314, 0.008706822991371155]\",\n          \"[-0.00985274463891983, -0.005485823843628168, 0.01965433917939663, -0.0197310633957386, -0.009596995078027248, 0.03214769437909126, 0.009807988069951534, -0.012314331717789173, -0.0037978780455887318, 0.011112309992313385, -0.012135307304561138, -0.014488201588392258, -0.01741653122007847, -0.01782573014497757, -0.0022985474206507206, 0.012288756668567657, 0.029232151806354523, -0.01898938976228237, 0.02150852233171463, -0.038081079721450806, 0.002520729787647724, 0.003526144428178668, -0.0016719615086913109, -0.010268337093293667, -0.026009710505604744, -0.010818198323249817, 0.01364422868937254, -0.023247618228197098, 0.01780015602707863, -0.04232652112841606, 0.01824771799147129, -0.01607384718954563, -0.001280345255509019, -0.016265660524368286, -0.030050549656152725, -0.007928230799734592, -0.007985773496329784, -0.007109832484275103, 0.009494694881141186, 0.0019980419892817736, 0.004309376701712608, -0.002989070722833276, -0.0036955783143639565, -0.01635517179965973, -0.02948790229856968, 0.012058582156896591, -0.0320453941822052, -0.02511458843946457, -0.032275568693876266, -0.002057183999568224, 0.030050549656152725, 0.021214410662651062, -0.005674439016729593, -0.021674757823348045, -0.0016200123354792595, -0.001309916260652244, 0.0026853682938963175, 0.014142939820885658, -0.0001769266091287136, -0.009501089341938496, 0.015114787966012955, -0.010332274250686169, -0.025741172954440117, -0.002662990242242813, 0.0009830365888774395, 0.027262883260846138, 0.004939159844070673, 0.001665567746385932, -0.016534196212887764, -0.007761993445456028, 0.018388379365205765, 0.0020619791466742754, -0.016598133370280266, -0.012160882353782654, 0.0508941225707531, -0.01788966730237007, 0.003618853632360697, -0.03138044849038124, 0.0014545745216310024, -0.004347739275544882, 0.015114787966012955, -0.0032000639475882053, -0.034372713416814804, 0.003177685895934701, 0.014641650952398777, 0.020856359973549843, -0.0034941756166517735, 0.030331874266266823, -0.003960918169468641, -0.002894763136282563, -0.024858837947249413, 0.020575037226080894, 0.005965353921055794, -0.013823253102600574, 0.005418689455837011, 0.006943595129996538, 0.0011844392865896225, 0.04163599759340286, -0.019539251923561096, -0.0017678674776107073, -0.027237307280302048, -0.0007980179507285357, -0.0025862655602395535, -0.01748046837747097, 0.011182640679180622, 0.0037882875185459852, -0.008056105114519596, 0.009884713217616081, -0.0037563187070190907, -0.014219664968550205, -0.007947411388158798, 0.012570081278681755, -0.002407240914180875, -0.017237506806850433, -0.018656916916370392, -0.014437051489949226, 0.004967931658029556, -0.03718595951795578, 0.0005610501975752413, -0.017736218869686127, 0.01129133440554142, 0.025012288242578506, -0.007039501331746578, -0.005262043327093124, 0.0073975506238639355, 0.01329896692186594, -0.023452216759324074, -0.010377030819654465, -0.0159587599337101, -0.01314551755785942, 0.04102219641208649, 0.01930907741189003, 0.0025814701803028584, 0.005869447719305754, -0.024922775104641914, 0.03432156518101692, -0.021009810268878937, 0.014769526198506355, -0.012851405888795853, -0.007608543615788221, 0.028899678960442543, 0.014513776637613773, -0.007992167957127094, -0.01887430250644684, -2.4276208932860754e-05, 0.006441687233746052, 0.03130372241139412, 0.0058119045570492744, 0.00765329971909523, 0.0009294890915043652, 0.03982017561793327, -0.026802534237504005, 0.013682590797543526, 0.009322063997387886, 0.008567603304982185, 0.006246678065508604, -0.005952566396445036, 0.015856461599469185, -0.009520269930362701, -0.0364442840218544, 0.0027413135394454002, 0.03097124770283699, 0.01599712297320366, -0.0014553737128153443, -0.014219664968550205, 0.01604827307164669, -0.0028883693739771843, 0.03201982006430626, 0.025715598836541176, -0.0020172230433672667, -0.02401486597955227, 0.0024216268211603165, -0.0327359177172184, -0.0118284085765481, -0.0019516873871907592, 0.028260305523872375, 0.017020119354128838, -0.00909828394651413, -0.0009934264235198498, -0.016994545236229897, -0.040996622294187546, 0.013401267118752003, 0.015882035717368126, 0.025268036872148514, -0.007761993445456028, -0.007992167957127094, 0.002877180464565754, -0.01934743858873844, 0.0023912566248327494, 0.007231313269585371, 0.008516454137861729, 0.02191772125661373, -0.017365381121635437, -0.01670043356716633, -0.6387596130371094, -0.014232452027499676, -0.026521209627389908, -0.018682491034269333, 0.007001138757914305, 0.02987152524292469, 0.0236568171530962, 0.0334264412522316, -0.004366920329630375, -0.0789242535829544, 0.0013330936199054122, 0.01475673820823431, 0.020920298993587494, -0.023912565782666206, -0.011975464411079884, -0.0111378850415349, -0.0006641491199843585, 0.0030402205884456635, 0.015396111644804478, 0.006182740908116102, -0.004507582634687424, 0.022684969007968903, -0.009839956648647785, -0.025025075301527977, 0.016086634248495102, 0.007941017858684063, -0.0006517612491734326, -0.022339707240462303, -0.020971447229385376, 0.021419009193778038, -0.0174676813185215, 0.01019161194562912, -0.002987472340464592, 0.032249994575977325, 0.04401446506381035, -0.02301744371652603, -0.0011101120617240667, 0.00473775714635849, -0.014181301929056644, 0.037646304816007614, -0.012512537650763988, 0.0193218644708395, 0.02664908394217491, -0.019385801628232002, -0.007435912732034922, -0.006630302406847477, 0.03028072416782379, -0.013132729567587376, -0.003845831146463752, 0.008484485559165478, -0.00873384065926075, -0.02698155865073204, -0.02478211373090744, -0.015869248658418655, 0.0010629582684487104, -0.00281803822144866, 0.037697456777095795, -0.0006957182195037603, -0.015242662280797958, -0.009245339781045914, 0.006208315957337618, 0.018017543479800224, 0.00784511212259531, 0.021048173308372498, 0.005709604360163212, 0.048285480588674545, -0.030050549656152725, -0.0037019720766693354, -0.006732602138072252, -0.015217087231576443, 0.010907710529863834, 0.002798857167363167, -0.028132429346442223, 0.010274730622768402, -0.017199145630002022, 0.010153249837458134, 0.04708345979452133, 0.010882135480642319, 0.01678994670510292, 0.00526843685656786, -0.006604727357625961, -0.017007332295179367, -0.03434713929891586, 0.010830985382199287, 0.0050190812908113, -0.015255449339747429, -0.016163360327482224, -0.017327019944787025, -0.008152010850608349, 0.0027317230124026537, 0.0009574617142789066, 0.020511098206043243, -0.0018701672088354826, -0.005150152835994959, -0.005060640629380941, 0.028183579444885254, -0.023976502940058708, 0.012486962601542473, -0.009290095418691635, -0.020178625360131264, 0.019935663789510727, -0.04265899583697319, 0.006636695936322212, -0.02037043683230877, 0.03061319887638092, -0.011815620586276054, -0.008490879088640213, 0.03066434897482395, 0.03654658421874046, -0.015319387428462505, 0.006332993507385254, 0.01384882815182209, 0.007851505652070045, -0.015255449339747429, -0.01127215288579464, -0.025651661679148674, 0.021943295374512672, 0.006355371791869402, 0.0076277246698737144, -0.024692600592970848, 0.023170892149209976, 0.0008487682207487524, 0.001609622617252171, -0.028183579444885254, 0.033017244189977646, 0.008682690560817719, -0.0197310633957386, -0.018746428191661835, -0.02074127271771431, 0.017569981515407562, 0.011380846612155437, 0.007966592907905579, 0.0189126655459404, -0.014296389184892178, 0.009290095418691635, -0.012729925103485584, 0.029053127393126488, -0.0035101601388305426, 0.001316310022957623, -0.02038322389125824, -0.004635457415133715, 0.004724969621747732, 0.024961138144135475, 0.006585546303540468, -0.009903893806040287, -0.030741073191165924, -0.002656596479937434, 0.006981957703828812, -0.023401066660881042, 0.006777358241379261, -0.015344961546361446, -0.0029714880511164665, -0.008254311047494411, -0.0034686008002609015, 0.011265759356319904, 0.01439868938177824, 0.009462726302444935, -0.00042638214654289186, -0.015153150074183941, 0.013823253102600574, 0.0007029111729934812, 0.02877180278301239, 0.00985274463891983, -0.020344862714409828, -0.003516553668305278, -0.02664908394217491, 0.03022957593202591, 0.013580291531980038, 0.015677435323596, -0.024935564026236534, 0.005137365311384201, -0.04076644778251648, -0.001695937942713499, 0.004430857952684164, -0.00418789591640234, 0.032991666346788406, 0.015434473752975464, -0.010869348421692848, 0.0033343322575092316, -0.003067394020035863, 0.013810466043651104, -0.023963715881109238, -0.02992267534136772, -0.0096289636567235, 0.03902735188603401, 0.0019996403716504574, 0.02081799879670143, 0.01237187534570694, 0.0061188037507236, 0.00392894959077239, -0.0057351794093847275, -0.0022442007903009653, 0.017365381121635437, 0.007953804917633533, -0.010888529010117054, 0.010204399935901165, 0.0028116446919739246, 0.013593078590929508, -0.0051757278852164745, 0.01934743858873844, 0.009782413020730019, -0.002854802180081606, 0.014245240017771721, 0.009213371202349663, 0.001659173984080553, -0.026725808158516884, 0.0035421287175267935, 0.0018669703276827931, 0.005092609208077192, 0.0043924953788518906, 0.010901317000389099, -0.044577114284038544, -0.009354033507406712, -0.00909828394651413, -0.006425702944397926, 0.023823052644729614, 0.007589362561702728, -0.0022873582784086466, -0.028311455622315407, 0.032966092228889465, 0.013593078590929508, 0.004529960453510284, 0.005505004897713661, -0.009501089341938496, -0.02772323042154312, -0.008650721982121468, -0.00263421842828393, 0.02075406163930893, 0.020958660170435905, 0.0010237967362627387, -0.015204300172626972, 0.0030290314462035894, -0.013797678053379059, 0.02584347315132618, -0.02515294961631298, 0.014692801050841808, 0.00927091483026743, -0.0200123880058527, 0.02405322715640068, 0.01713520660996437, -0.015779735520482063, 0.006566364783793688, 0.014680013991892338, -0.013337329030036926, 0.006432096473872662, -0.01817099191248417, 0.023094167932868004, 0.007333613000810146, 0.00016264061559922993, -0.02553657442331314, 0.005536973942071199, -0.00797937996685505, 0.0028004555497318506, 0.008471697568893433, 0.0017311035189777613, -0.012621231377124786, 0.001396231702528894, -0.005946172866970301, -0.021777058020234108, 0.004702591337263584, 0.0001276748807867989, 0.03158504515886307, 0.03250574320554733, 0.01898938976228237, 0.0061571658588945866, 0.010664748959243298, -0.0016335990512743592, -0.0071737696416676044, -0.0002151891094399616, -0.027518631890416145, -0.011431996710598469, 0.00727606937289238, 0.023119742050766945, -0.013426841236650944, 0.016316808760166168, 0.006460868287831545, 0.018350016325712204, -0.0182221420109272, 0.031457170844078064, -0.005856660660356283, -0.012582868337631226, -0.03386121615767479, 0.016892245039343834, 0.024283401668071747, -0.018861515447497368, -0.011905132792890072, -0.00803053006529808, 0.000551459554117173, -0.0014034246560186148, 0.033375293016433716, 0.005658454727381468, -0.009501089341938496, 0.016943395137786865, -0.011956282891333103, -0.01855461671948433, 0.012077763676643372, 0.026802534237504005, -0.01791524328291416, -0.018708067014813423, -0.012365481816232204, -0.02548542432487011, -0.009705688804388046, 0.002717337105423212, -0.03636755794286728, -0.01565186120569706, 0.023823052644729614, -0.0007085056859068573, -0.038643728941679, -0.0012907350901514292, -0.020907510071992874, -0.006512018386274576, 0.00363803468644619, -0.02882295288145542, 0.007941017858684063, -0.011035584844648838, -0.013235029764473438, -0.0393342524766922, -0.0026134387589991093, 0.023068591952323914, -0.007359188050031662, -0.030024975538253784, -0.005971747916191816, -0.011240184307098389, -0.0020667745266109705, 0.03872045502066612, 0.034781914204359055, -0.02769765630364418, 0.006879657972604036, 0.01635517179965973, 0.01749325729906559, 0.003331135492771864, -0.0372626818716526, 0.020869148895144463, 0.0010613598860800266, -0.006873263977468014, -0.00875302217900753, 0.009059920907020569, 0.013222241774201393, -0.004293392412364483, 0.013746527954936028, 0.0334264412522316, -0.030050549656152725, 0.007340006995946169, -0.009795201011002064, -0.031099122017621994, -0.015677435323596, 0.0444236658513546, 0.016559772193431854, -0.018452316522598267, -0.0363164097070694, -0.009059920907020569, 0.00926452036947012, 0.016086634248495102, -0.03219884634017944, 0.0029171411879360676, -0.013618653640151024, 0.00019450938270892948, 0.016508622094988823, -0.0070139262825250626, -0.0226210318505764, 0.017953606322407722, 0.029973825439810753, 0.03452616557478905, -0.006016504019498825, 0.009929468855261803, 0.012800255790352821, 0.022812843322753906, -0.009891106747090816, 0.007947411388158798, -0.003912965301424265, -0.008152010850608349, 0.004539551213383675, -0.012314331717789173, -0.004760134965181351, 0.03094567358493805, -0.006364962551742792, -0.004248636309057474, 0.015766948461532593, -0.011252972297370434, 0.01681552082300186, 0.03056204877793789, -0.019488101825118065, 0.00036644088686443865, 0.03063877485692501, 0.00581829808652401, -0.013119942508637905, 0.006070850417017937, 0.0031073547434061766, -0.005792723037302494, -0.022748906165361404, -0.0026294232811778784, -0.03961557522416115, -0.02008911222219467, -0.0039161620661616325, -0.009430757723748684, -0.013107155449688435, -0.0066878460347652435, -0.019181203097105026, 0.02984595112502575, 0.02698155865073204, 0.012467782013118267, -0.007941017858684063, -5.52458586753346e-05, -0.007455093786120415, -0.009513876400887966, -0.010428179986774921, -0.009136646054685116, 0.0005406701238825917, -0.0021019401028752327, -0.001260364893823862, 0.006796539295464754, 4.64045224362053e-05, -0.026853682473301888, 0.019424164667725563, -0.020920298993587494, 0.00417830515652895, 0.010703111067414284, 0.003980099689215422, -0.012154488824307919, 0.012857799418270588, -0.012141700834035873, 0.002957102144137025, -0.011329696513712406, -0.013324541971087456, 0.005767147988080978, -0.06004995107650757, -0.01571579836308956, -0.003104157978668809, 0.013183879666030407, 0.0013818457955494523, -0.014769526198506355, 0.026802534237504005, -0.0061060162261128426, -0.019040539860725403, 0.018490679562091827, -0.01785130612552166, 0.003989689983427525, -0.013247816823422909, 0.016610920429229736, 0.014424264430999756, -0.009533057920634747, 0.019743850454688072, -0.009379607625305653, 0.0237207543104887, 0.023874202743172646, -0.015242662280797958, 0.036623310297727585, 0.025945773348212242, -0.019922874867916107, 0.014015065506100655, -0.028132429346442223, -0.012838617898523808, -0.01056884229183197, 0.006460868287831545, 0.012192850932478905, 0.002009230898693204, -0.016853883862495422, -0.02288956753909588, -0.013311754912137985, -0.017557194456458092, -0.01887430250644684, 0.016968971118330956, -0.005210893228650093, 0.017940817400813103, 0.013529141433537006, -0.021994445472955704, -0.02879737876355648, -0.007090650964528322, -0.009366820566356182, -0.026853682473301888, 0.003121740650385618, 0.004216667730361223, -0.01824771799147129, 0.021649183705449104, 0.017288656905293465, -0.021329497918486595, -0.006598333828151226, 0.015549561008810997, -0.007039501331746578, -0.048976004123687744, 0.0021498932037502527, -0.0030721891671419144, 0.0004751343803945929, -0.011611021123826504, 0.03641870990395546, 0.01147675234824419, 0.025702811777591705, 0.022723332047462463, -0.014066214673221111, 0.004881616216152906, -0.000526683870702982, -0.015229874290525913, -0.0026150373741984367, 0.03283821791410446, 0.028925253078341484, 0.014488201588392258, 0.018772004172205925, 0.008791384287178516, 0.016994545236229897, 0.020587824285030365, -0.003519750665873289, 0.003292773151770234, -0.014500988647341728, 0.0020236168056726456, -0.011956282891333103, -0.009513876400887966, -0.000679733871947974, 0.01635517179965973, -0.0024168314412236214, -0.021278347820043564, 0.00619552843272686, 0.005591320339590311, -0.019219564273953438, -0.023835841566324234, 0.010447361506521702, -0.019948450848460197, 0.028567204251885414, -0.007378369104117155, 0.013273391872644424, -0.0016479849582538009, 0.0066111208871006966, -0.01530659943819046, -0.036239683628082275, 0.030331874266266823, 0.018311655148863792, 0.004239045549184084, -0.003538931952789426, -0.006451277527958155, 0.0035485224798321724, 0.0036220503970980644, -0.011700533330440521, 0.00411117123439908, 0.00572239188477397, 0.016930608078837395, -0.025753961876034737, -0.03099682368338108, 0.011195428669452667, -0.039538852870464325, -0.019066115841269493, 0.0060644568875432014, -0.0237207543104887, 0.011700533330440521, 0.010504905134439468, -0.03166177123785019, -0.009443545714020729, 0.01274910569190979, 0.04526763781905174, 0.024488002061843872, 0.024897200986742973, 0.008056105114519596, -0.028158005326986313, -0.01186037715524435, 0.00985274463891983, 0.016841094940900803, -0.021252771839499474, 0.018439529463648796, 0.007761993445456028, 0.011726108379662037, -0.01777458004653454, 0.006988351233303547, -0.005738376174122095, -0.044219065457582474, -0.027953404933214188, 0.02045994997024536, 0.001444983878172934, -0.011438390240073204, 0.0017231113743036985, 0.012333513237535954, -0.008669903501868248, 0.01678994670510292, -0.008861715905368328, 0.006125197280198336, -0.021725907921791077, -0.02223740704357624, -0.027058282867074013, 0.011598234064877033, 0.009507482871413231, -0.015792522579431534, -0.007442306727170944, -0.014219664968550205, -0.005648863967508078, -0.017723431810736656, -0.008158404380083084, 0.0018701672088354826, -0.014360327273607254, 0.007806749548763037, -0.036572158336639404, 0.013593078590929508, 0.025702811777591705, -0.007698055822402239, 0.009021558798849583, 0.013388479128479958, -0.004529960453510284, 0.02223740704357624, -0.003014645539224148, 0.010607205331325531, -0.012934524565935135, 0.02077963575720787, 0.008420547470450401, 0.0124102383852005, 0.012282363139092922, 0.003257607575505972, 0.00013446822413243353, 0.013183879666030407, 0.03140602260828018, 0.024155527353286743, -0.011035584844648838, 0.011885952204465866, -0.01708405837416649, 0.012691562063992023, -0.015856461599469185, -0.0050478531047701836, 0.009999800473451614, 0.02836260385811329, -0.05283781886100769, -0.022135106846690178, 0.0010693520307540894, 0.02874622866511345, -0.01328617986291647, 0.003024236299097538, 0.008145617321133614, 0.03603508695960045, -0.014270815066993237, -0.00895762164145708, -0.025664448738098145, -0.0015568742528557777, -0.00682211434468627, 0.010997222736477852, -0.007301644422113895, -0.004801694303750992, 0.042761292308568954, -0.00527802761644125, -0.028209155425429344, -0.01314551755785942, 0.015165937133133411, -0.014986912719905376, 0.009596995078027248, 0.004584307316690683, -0.00856120977550745, -0.004466023296117783, -0.010325880721211433, -0.014526563696563244, -0.001028591999784112, 0.010134068317711353, 0.0047281663864851, 0.003436631988734007, 0.0178129430860281, -0.0017470878083258867, 0.00033507164334878325, 0.017237506806850433, -0.01257647480815649, -0.007109832484275103, -0.008068892173469067, -0.018413953483104706, 0.0009023157181218266, 0.009756837971508503, -0.002245799172669649, -0.013733740895986557, -0.03132929652929306, 0.007550999987870455, -0.007819537073373795, 0.0007804351625964046, 0.020690124481916428, -0.013963915407657623, -0.032224420458078384, 0.023592878133058548, 0.03649543598294258, -0.005057443864643574, -0.0030386222060769796, -0.02044716104865074, 0.005546564236283302, -0.015140362083911896, -0.037723030894994736, -0.019232353195548058, 0.0026981558185070753, 0.026124797761440277, 0.019782213494181633, -0.010421786457300186, -0.0319175198674202, 0.01529381237924099, -0.012001038528978825, 0.018413953483104706, -0.020830785855650902, 0.007550999987870455, -0.005124577786773443, -0.019897300750017166, -0.01290894951671362, 0.015204300172626972, 0.021022597327828407, -0.0034142539370805025, -0.001450578449293971, 0.007710843347012997, -0.00031549081904813647, -0.006380946841090918, 0.00774920592084527, 0.003944933880120516, -0.012979280203580856, 0.017250293865799904, -0.021649183705449104, 0.00021638793987222016, 0.008305460214614868, -0.0015688624698668718, -0.01475673820823431, -0.023771904408931732, -0.003019440919160843, 0.01889987848699093, 0.021585246548056602, 0.022736119106411934, 0.0009294890915043652, -0.011323302984237671, -0.02143179625272751, 0.006553577724844217, 0.019053326919674873, -0.007742811925709248, 0.018375592306256294, 0.03460288792848587, -0.02618873491883278, 0.008893684484064579, -0.03654658421874046, -0.010511298663914204, 0.013209454715251923, -0.004708985332399607, 0.03201982006430626, -0.015882035717368126, 0.0103834243491292, 0.00804971158504486, 0.011201822198927402, -0.005223680753260851, 0.0020475934725254774, 0.012499750591814518, -0.0029970628675073385, -0.0007512637530453503, -0.019206777215003967, -0.02107374742627144, 0.021930508315563202, -0.022493157535791397, -0.02334991656243801, -0.0193218644708395, 0.024513576179742813, -0.014347539283335209, -0.024577513337135315, -0.04150812327861786, 0.023938139900565147, 0.032275568693876266, -0.012454994022846222, 0.0197310633957386, -0.01969270035624504, -0.01574137434363365, -0.00773002440109849, 0.0011564666638150811, 0.03317069262266159, -0.003992886748164892, -0.026444483548402786, 0.03283821791410446, -0.008471697568893433, 0.02191772125661373, -0.004491598345339298, 0.0007336809649132192, 0.019756639376282692, -0.0005238865851424634, 0.23835840821266174, -0.01609942317008972, -0.004050430376082659, 0.012800255790352821, 0.010466543026268482, 0.017301443964242935, 0.02879737876355648, 0.005137365311384201, -0.0026613918598741293, 0.0006597534520551562, -0.00998701248317957, -0.031047973781824112, -0.020268136635422707, 0.011623809114098549, -0.010824591852724552, -0.022096745669841766, -0.004453235771507025, -0.003220843616873026, 0.012499750591814518, -0.057748205959796906, -0.003398269647732377, -0.0170584823936224, -0.001181242405436933, -0.028209155425429344, 0.03414253890514374, 0.023043017834424973, -0.011425603181123734, -0.028337029740214348, 0.018823152408003807, 0.003526144428178668, 0.004664229229092598, -0.02690483257174492, -0.010645567439496517, 0.005613698624074459, -0.0019612780306488276, -0.007205738220363855, -0.006061260122805834, 0.019500888884067535, 0.0006965174106881022, 0.030843373388051987, 0.0018318048678338528, 0.028950829058885574, -0.004609882365912199, 0.011540690436959267, 0.018388379365205765, 0.012256788089871407, -0.0026486043352633715, -0.006253072060644627, -0.007787568029016256, 0.011777258478105068, -0.00799856148660183, -0.006700633559376001, 0.013593078590929508, 0.025664448738098145, -0.008989590220153332, 0.007250494323670864, 0.008765809237957, 0.004293392412364483, -0.009085495956242085, -0.024590302258729935, 0.01851625367999077, 0.019373014569282532, -0.029027553275227547, 0.00572239188477397, -0.019922874867916107, 0.011674958281219006, -0.03196867182850838, -0.0011292933486402035, 0.00436052680015564, -0.030459748581051826, -0.009251733310520649, -0.028976403176784515, -0.016905033960938454, -0.006077244412153959, -0.011796439066529274, -0.00031768865301273763, 0.0051757278852164745, -0.021291134878993034, 0.0017231113743036985, 0.031175848096609116, 0.023183679208159447, -0.0069947452284395695, 0.00296349567361176, 0.038643728941679, -0.020575037226080894, -0.020690124481916428, 0.04288917034864426, 0.0012883374001830816, 0.003567703766748309, -0.019104477018117905, 0.011930707842111588, 0.04542108625173569, -0.01256368774920702, -0.010293912142515182, 0.008196767419576645, -0.004708985332399607, 0.03457731381058693, 0.018746428191661835, 0.0011508720926940441, -0.0030322284437716007, -0.013247816823422909, -0.014488201588392258, 0.01274910569190979, -0.0045555355027318, 0.0029171411879360676, 0.012263181619346142, 0.006553577724844217, 0.019373014569282532, 0.02113768458366394, -0.0035485224798321724, 0.004926372319459915, -0.02003796212375164, 0.006534396205097437, 0.02040879987180233, -0.0071929506957530975, -0.01638074778020382, 0.009948650375008583, -0.006355371791869402, -0.019846150651574135, -0.0036572159733623266, 0.0186313409358263, 0.010939679108560085, -0.00927091483026743, 0.01329896692186594, 0.004993506241589785, -0.015165937133133411, -0.014104577712714672, 0.0035900818184018135, 0.012064976617693901, -0.038848329335451126, 0.005255649331957102, -0.010594417341053486, 0.03434713929891586, -0.0024903593584895134, 0.0010853363201022148, -0.0004103977989871055, 0.0037275468930602074, 0.023209255188703537, -0.01474395114928484, 0.0031153468880802393, -0.008062498643994331, -0.014270815066993237, 0.007966592907905579, 0.022454794496297836, -0.018695278093218803, -0.04066414758563042, -0.0037339406553655863, 0.005789526272565126, -0.011617414653301239, 0.0030657954048365355, -0.040996622294187546, -0.00964175071567297, -0.010888529010117054, -0.02077963575720787, 0.029999401420354843, -0.0019053327850997448, -0.029436752200126648, 0.007563787512481213, 0.013439629226922989, 0.010504905134439468, -0.01962876319885254, 0.009373214095830917, 0.012467782013118267, -0.022058382630348206, 0.00013027233944740146, -0.002744510304182768, -0.1597921997308731, 0.005495414603501558, 0.016866670921444893, -0.0160226970911026, 0.011591839604079723, 0.008554816246032715, 0.008363003842532635, 0.027058282867074013, -0.030024975538253784, -0.0035740972962230444, 0.015050849877297878, -0.03253132104873657, -0.03278706967830658, -0.015153150074183941, -0.013107155449688435, -0.00947551429271698, -0.010485723614692688, 0.006147575564682484, 0.023401066660881042, 0.014871825464069843, 0.013119942508637905, -0.020229775458574295, 0.017646705731749535, 0.011431996710598469, 0.021112110465765, 0.014769526198506355, 0.007787568029016256, 0.029794801026582718, -0.02626545913517475, -0.01638074778020382, 0.011316909454762936, -0.004523566924035549, 0.003014645539224148, -0.007678874768316746, 0.03388679027557373, 0.009136646054685116, 0.011246578767895699, -0.007020320277661085, -0.014986912719905376, 0.022160682827234268, 0.005309996195137501, 0.018848728388547897, 0.02187935821712017, -0.0010685528395697474, -0.01824771799147129, 0.010268337093293667, 0.0083566103130579, -0.0009902295423671603, 0.0035836880560964346, 0.003746728179976344, 0.004999900236725807, -0.030510898679494858, 0.03132929652929306, 0.0073975506238639355, 0.026853682473301888, -0.009181402623653412, 0.026444483548402786, 0.02438570186495781, 0.001659173984080553, -0.0023496972862631083, -0.017390957102179527, -0.02003796212375164, 0.0023097365628927946, 0.001756678451783955, 0.0023289176169782877, 0.015485623851418495, 0.017263082787394524, -0.02401486597955227, -0.03964115306735039, 0.003061000257730484, -0.01364422868937254, -0.03383564203977585, -0.0031201422680169344, -0.03020399995148182, 0.0015297009376809, 0.0002797258784994483, -0.006630302406847477, -0.007116226013749838, 0.00454594474285841, 0.03237786889076233, 0.01745489425957203, 0.02363124117255211, -0.03306839242577553, -0.01369537878781557, 0.0013179084053263068, 0.020549461245536804, -0.0255749374628067, 0.011834802106022835, -0.0056840297766029835, -0.012160882353782654, 0.028234729543328285, -0.02479490078985691, -0.02444963902235031, -0.01970548927783966, 0.012141700834035873, 0.022058382630348206, 0.015191512182354927, 0.010971647687256336, 0.019104477018117905, -0.01826050505042076, 0.009277308359742165, -0.0013482787180691957, 0.012116125784814358, 0.015907609835267067, 0.006345781031996012, 0.0033087574411183596, -0.007915442809462547, -0.0006589542026631534, 0.012448600493371487, -0.006473655812442303, -0.03501208871603012, -0.021636396646499634, 0.024705389514565468, 0.023490579798817635, -0.027544206008315086, 0.028720654547214508, 0.0021387040615081787, -0.025319186970591545, -0.001935702981427312, -0.027109432965517044, 0.059589602053165436, -0.011937101371586323, -0.004143139813095331, -0.0240660160779953, -0.02948790229856968, -0.02404044009745121, -0.08869387954473495, -0.027851106598973274, -0.022058382630348206, -0.008113648742437363, 0.04961537569761276, 0.03242902085185051, -0.005632879678159952, -0.0004327758797444403, 0.003091370454058051, 0.0043829046189785, -0.011949889361858368, -0.0053547522984445095, 0.0013362903846427798, -0.003801075043156743, -0.004760134965181351, -0.008094467222690582, 0.009724869392812252, -0.02912985347211361, -0.02373354136943817, 0.021009810268878937, -0.01680273376405239, 0.0030466143507510424, 0.009788806550204754, -0.007327219471335411, 0.0102107934653759, -0.026828108355402946, -0.017659492790699005, 0.012966493144631386, 0.01290894951671362, 0.013247816823422909, -0.010792623274028301, -0.004114367999136448, -0.019743850454688072, -0.014769526198506355, 0.000734480214305222, -0.006304221693426371, -0.014871825464069843, 0.02478211373090744, 0.00034066615626215935, -0.042479969561100006, 0.017378170043230057, -0.011674958281219006, -0.004513976164162159, -0.01824771799147129, 0.004945553373545408, 0.007710843347012997, -0.0028755818493664265, -0.007365581579506397, 0.005866250954568386, 0.009449939243495464, -0.014731163159012794, 0.0030753861647099257, -0.007487062830477953, -0.004539551213383675, 0.017646705731749535, 0.003794681280851364, 0.0059557631611824036, 0.013797678053379059, -0.019193990156054497, -0.006543986964970827, -0.00048632340622134507, 0.00013946332910563797, -0.011981857940554619, 0.019475314766168594, 0.0189766027033329, 0.01364422868937254, 0.014117364771664143, -0.003692381549626589, 0.009993406012654305, 0.01962876319885254, -0.010313092730939388, 0.002117924392223358, 0.0008375791949220002, -0.007071469910442829, -0.020984236150979996, 0.004820875357836485, -0.00968011375516653, -0.009795201011002064, 0.005898219533264637, -0.04708345979452133, -0.003893784014508128, -0.017557194456458092, 0.013426841236650944, -0.005575336050242186, 0.006313812453299761, -0.005463446024805307, -0.02075406163930893, 0.014079002663493156, 0.00465144170448184, 0.010166036896407604, -0.0019117265474051237, 0.04378429055213928, 0.010319487191736698, -0.0006789346225559711, -0.0015400906559079885, 0.003992886748164892, -0.01365701574832201, -0.010249155573546886, 0.045932587236166, 0.02662350982427597, -0.012192850932478905, 0.004568323027342558, -0.050868548452854156, 0.021725907921791077, -0.02764650620520115, -0.010114887729287148, -0.002057183999568224, -0.007244100794196129, 0.02946232631802559, -0.005591320339590311, 0.005978141445666552, -0.0062115127220749855, -0.02629103511571884, 0.022748906165361404, 0.013874403201043606, 0.007742811925709248, -0.03685348480939865, -0.0207668486982584, -0.007557393983006477, 0.019603189080953598, -0.010498511604964733, 0.012103338725864887, -0.018107054755091667, -0.015856461599469185, 0.009258126839995384, 0.017339807003736496, -0.01965433917939663, 0.019181203097105026, -0.037339407950639725, 0.014974125660955906, -0.02662350982427597, -0.009443545714020729, 0.012192850932478905, -0.021930508315563202, -0.021265558898448944, 0.03716038167476654, 0.0019916482269763947, 0.008369398303329945, 0.021598033607006073, -0.013017642311751842, 0.027109432965517044, 0.016841094940900803, -0.039487700909376144, -0.02805570513010025, 0.027569781988859177, -0.03488421440124512, 0.013235029764473438, 0.0035069631412625313, -0.030689923092722893, 0.01549841184169054, 0.02879737876355648, 0.006668664515018463, 0.013235029764473438, 0.033375293016433716, -0.010677536018192768, -0.023247618228197098, 0.0033055604435503483, -0.03539571166038513, 0.008701872080564499, -0.0050286720506846905, 0.02764650620520115, -0.03352874144911766, 0.012461387552320957, -0.01510199997574091, 0.020894723013043404, 0.011419208720326424, 0.01785130612552166, 0.004629063419997692, 0.021725907921791077, -0.022416431456804276, 0.012761893682181835, -0.04255669564008713, -0.0015744570409879088, 0.0068157208152115345, 0.003398269647732377, 0.013580291531980038, -0.0013762512244284153, 0.027978980913758278, -0.025702811777591705, 0.004696197807788849, -0.012672381475567818, 0.00927091483026743, 0.005514595657587051, 0.006681452039629221, -0.00710343848913908, 0.009807988069951534, 0.045932587236166, 0.02072848565876484, 0.005357949063181877, 0.004302983172237873, 0.024321764707565308, 0.014654438942670822, -0.020894723013043404, 0.0225570946931839, -0.0016927410615608096, -0.005604107864201069, 0.016470259055495262, 0.007346400525420904, -0.009392395615577698, -0.01861855387687683, 0.011700533330440521, 0.020690124481916428, 0.010146856307983398, 0.007218525744974613, -0.007410337682813406, -0.021227197721600533, -0.009354033507406712, 0.02581789903342724, -0.008804172277450562, -0.026316609233617783, 0.024257827550172806, -0.00581829808652401, 0.006809326820075512, -0.011988251470029354, 0.014897400513291359, 0.014091789722442627, -0.027083856984972954, -0.004053627606481314, -0.021316708996891975, -0.013899978250265121, -0.02075406163930893, 0.03537013754248619, 0.020830785855650902, 0.014334752224385738, 0.03248016908764839, 0.00043077784357592463, 0.014168514870107174, 0.0059365821070969105, 0.003177685895934701, -0.0071418010629713535, 0.001935702981427312, -0.009430757723748684, 0.0035964753478765488, -0.01056884229183197, -0.011732501909136772, -0.005067034158855677, 0.0007756398990750313, -0.009558632969856262, -0.01571579836308956, 0.014922975562512875, -0.020626185461878777, 0.08542028814554214, 0.013477991335093975, -0.0007332813693210483, 0.01529381237924099, -0.013899978250265121, 0.026367759332060814, 0.010664748959243298, -0.014258027076721191, -0.01893823966383934, 0.008919259533286095, 0.013324541971087456, -0.02299186773598194, 0.014232452027499676, -0.0393342524766922, -0.030306300148367882, 0.010287518613040447, -0.006937201600521803, -0.0008471697801724076, -0.010319487191736698, 0.004619473125785589, 0.012275969609618187, -0.0011149074416607618, 0.008631541393697262, -0.005099003203213215, 0.012608443386852741, -0.015945972874760628, 0.02989710122346878, 0.011802833527326584, -0.011988251470029354, -0.017327019944787025, 0.02626545913517475, 0.006320205982774496, -0.019769426435232162, -0.018720854073762894, 0.018759215250611305, -0.007953804917633533, 0.00985274463891983, 0.006326599977910519, -0.0036540189757943153, 0.008612359873950481, -0.01022997498512268, 0.03127814829349518, -0.0244752150028944, -0.006326599977910519, 0.014974125660955906, -0.005869447719305754, 0.006393734365701675, 0.00964814517647028, -0.005789526272565126]\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"cosine_similarity\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.033062434045649396,\n        \"min\": 0.7651159604362697,\n        \"max\": 0.9401074237412784,\n        \"num_unique_values\": 420,\n        \"samples\": [\n          0.8193018014909341,\n          0.8882798140457474\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_custom[\"pergunta_embedding\"] = df_custom[\"pergunta_embedding\"].apply(eval).apply(np.array)\n",
        "df_custom[\"chunk_embedding\"] = df_custom[\"chunk_embedding\"].apply(eval).apply(np.array)\n",
        "df_custom = df_custom.assign(cosine_similarity = df_custom.cosine_similarity.apply(lambda x: float(x)))"
      ],
      "metadata": {
        "id": "k7GoZPEnCJRy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#best_matrix = torch.load(f\"{path}best_matrix.zip\", map_location=\"cpu\")\n",
        "best_matrix = torch.load(f\"{path}best_matrix\", map_location=\"cpu\")\n",
        "apply_matrix_to_embeddings_dataframe(best_matrix, df_custom)\n",
        "df_custom.head(2)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 181
        },
        "id": "wspyggeM_hVs",
        "outputId": "c2de098f-43fa-4847-a98e-357cc4281010"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                            pergunta  \\\n",
              "0  Qual é o segredo que o Sr. e a Sra. Dursley te...   \n",
              "1  Qual era a cor da capa que o Sr. Dursley viu u...   \n",
              "\n",
              "                                            contexto  \\\n",
              "0  Os Dursley tinham tudo que queriam, mas tinham...   \n",
              "1  O Sr. Dursley não tolerava gente que andava co...   \n",
              "\n",
              "                   resposta  \\\n",
              "0   a existência dos Potter   \n",
              "1  uma capa verde-esmeralda   \n",
              "\n",
              "                                               chunk  label dataset  \\\n",
              "0  - CAPÍTULO UM - O menino que sobreviveu O Sr. ...      1   train   \n",
              "1  ante o caminho para a cidade ele não pensou em...      1   train   \n",
              "\n",
              "                                  pergunta_embedding  \\\n",
              "0  [0.022689364850521088, 0.0038326631765812635, ...   \n",
              "1  [0.005216388031840324, 0.0008732206770218909, ...   \n",
              "\n",
              "                                     chunk_embedding  cosine_similarity  \\\n",
              "0  [0.014419169165194035, 0.011491919867694378, -...           0.873794   \n",
              "1  [-0.002306348877027631, -0.0014758666511625051...           0.873999   \n",
              "\n",
              "                           pergunta_embedding_custom  \\\n",
              "0  [0.17453432, -0.60518175, -0.7326583, 0.358211...   \n",
              "1  [-0.62348986, -0.43539473, -0.32246643, -0.106...   \n",
              "\n",
              "                              chunk_embedding_custom  cosine_similarity_custom  \n",
              "0  [0.024289679, -0.73309463, -0.37434945, 0.1869...                  0.398477  \n",
              "1  [-0.28205645, -0.14895901, 0.066049635, 0.5515...                  0.479307  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-d30807b5-7294-4bbf-ac45-d82ee5366823\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>pergunta</th>\n",
              "      <th>contexto</th>\n",
              "      <th>resposta</th>\n",
              "      <th>chunk</th>\n",
              "      <th>label</th>\n",
              "      <th>dataset</th>\n",
              "      <th>pergunta_embedding</th>\n",
              "      <th>chunk_embedding</th>\n",
              "      <th>cosine_similarity</th>\n",
              "      <th>pergunta_embedding_custom</th>\n",
              "      <th>chunk_embedding_custom</th>\n",
              "      <th>cosine_similarity_custom</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Qual é o segredo que o Sr. e a Sra. Dursley te...</td>\n",
              "      <td>Os Dursley tinham tudo que queriam, mas tinham...</td>\n",
              "      <td>a existência dos Potter</td>\n",
              "      <td>- CAPÍTULO UM - O menino que sobreviveu O Sr. ...</td>\n",
              "      <td>1</td>\n",
              "      <td>train</td>\n",
              "      <td>[0.022689364850521088, 0.0038326631765812635, ...</td>\n",
              "      <td>[0.014419169165194035, 0.011491919867694378, -...</td>\n",
              "      <td>0.873794</td>\n",
              "      <td>[0.17453432, -0.60518175, -0.7326583, 0.358211...</td>\n",
              "      <td>[0.024289679, -0.73309463, -0.37434945, 0.1869...</td>\n",
              "      <td>0.398477</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Qual era a cor da capa que o Sr. Dursley viu u...</td>\n",
              "      <td>O Sr. Dursley não tolerava gente que andava co...</td>\n",
              "      <td>uma capa verde-esmeralda</td>\n",
              "      <td>ante o caminho para a cidade ele não pensou em...</td>\n",
              "      <td>1</td>\n",
              "      <td>train</td>\n",
              "      <td>[0.005216388031840324, 0.0008732206770218909, ...</td>\n",
              "      <td>[-0.002306348877027631, -0.0014758666511625051...</td>\n",
              "      <td>0.873999</td>\n",
              "      <td>[-0.62348986, -0.43539473, -0.32246643, -0.106...</td>\n",
              "      <td>[-0.28205645, -0.14895901, 0.066049635, 0.5515...</td>\n",
              "      <td>0.479307</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-d30807b5-7294-4bbf-ac45-d82ee5366823')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-d30807b5-7294-4bbf-ac45-d82ee5366823 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-d30807b5-7294-4bbf-ac45-d82ee5366823');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-a99c1c4c-dec0-4966-83fe-de84b57489b7\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-a99c1c4c-dec0-4966-83fe-de84b57489b7')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-a99c1c4c-dec0-4966-83fe-de84b57489b7 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "df_custom",
              "summary": "{\n  \"name\": \"df_custom\",\n  \"rows\": 420,\n  \"fields\": [\n    {\n      \"column\": \"pergunta\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 202,\n        \"samples\": [\n          \"amente evitara encarar Harry. Resmungou alguma coisa e lhe ofereceu mais um biscoito. Harry releu a not\\u00edcia. O cofre aberto na realidade fora esvaziado mais cedo naquele dia. Hagrid esvaziara o cofre setecentos e treze, se \\u00e9 que se podia chamar esvaziar algu\\u00e9m levar aquele pacotinho encalombado. Seria aquilo que os ladr\\u00f5es estavam procurando? Quando Harry e Rony voltaram ao castelo para jantar, tinham os bolsos pesados com os biscoitos que a educa\\u00e7\\u00e3o os impedira de recusar. Harry pensou que nenhuma das aulas a que assistira at\\u00e9 ali tinha-lhe dado tanto o que pensar quanto o ch\\u00e1 com R\\u00fabeo Hagrid. Ser\\u00e1 que Hagrid tinha apanhado o pacote bem na hora? Onde estava o pacote agora? Ser\\u00e1 que ele sabia alguma coisa de Snape que n\\u00e3o queria contar a Harry? - CAP\\u00cdTULO NOVE - O duelo \\u00e0 meia-noite Harry jamais acreditara que fosse encontrar um garoto que ele detestasse mais do que Duda, mas isto foi antes de conhecer Draco. Os alunos do primeiro ano da Grifin\\u00f3ria, por\\u00e9m, s\\u00f3 tinham uma aula com os da Sonserina, a de Po\\u00e7\\u00f5es, por isso n\\u00e3o precisavam aturar Draco muito tempo. Ou pelo menos, n\\u00e3o precisavam at\\u00e9 verem um aviso pregado no sala comunal de Grifin\\u00f3ria que fez todos gemerem. As aulas de voo come\\u00e7ariam na quinta-feira - e os alunos das duas casas aprenderiam juntos. - T\\u00edpico - disse Harry, desanimado. - \\u00c9 o que eu sempre quis. Fazer papel de palha\\u00e7o montado numa vassoura na frente do Draco. Ele estivera ansioso para aprender a voar, mais do que qualquer outra coisa. - Voc\\u00ea n\\u00e3o sabe se vai fazer papel de palha\\u00e7o - disse Rony, sensato. - Em todo o caso, sei que Draco vive falando que \\u00e9 bom em quadribol, mas aposto que \\u00e9 conversa fiada. Draco sem d\\u00favida falava muito de voos. Queixava-se em voz alta que os alunos do primeiro ano nunca entravam para o time de quadribol e se gabava em longas hist\\u00f3rias, que sempre pareciam terminar com ele escapando por um triz dos trouxas de helic\\u00f3pteros. Mas ele n\\u00e3o era o \\u00fanico: pelo que Simas Finnigan contava, ele passara a maior parte da inf\\u00e2ncia voando pelo campo montado numa vassoura. At\\u00e9 Rony contava para quem quisesse ouvir sobre a vez em que ele quase batera numa asa delta montado na velha vassoura de Carlinhos. Todos os garotos de fam\\u00edlias de bruxos falavam o tempo todo de quadribol. Rony j\\u00e1 tivera uma grande discuss\\u00e3o sobre futebol com Dino Thomas, que tamb\\u00e9m usava o dormit\\u00f3rio deles. Rony n\\u00e3o via nada excitante em um jogo em que ningu\\u00e9m podia voar e s\\u00f3 tinha uma bola. Harry surpreendera Rony cutucando o p\\u00f4ster em que Dino aparecia com o time de futebol de West Ham, tentando fazer os jogadores se mexerem. Neville nunca andara de vassoura na vida, porque a av\\u00f3 nunca o deixara chegar perto de uma. No fundo, Harry achava que ela estava cert\\u00edssima, porque Neville conseguira sofrer um n\\u00famero impressionante de acidentes mesmo com os dois p\\u00e9s no ch\\u00e3o. Hermione Granger estava quase t\\u00e3o nervosa quanto Neville com a ideia de voar. Isto n\\u00e3o era coisa que se aprendesse de cor em um livro - n\\u00e3o que ela n\\u00e3o tivesse tentado. No caf\\u00e9 da manh\\u00e3 de quinta-feira, deu um cansa\\u00e7o neles falando sobre macetes de voo que lera em um livro da biblioteca chamado Quadribol atrav\\u00e9s dos s\\u00e9culos. Neville praticamente se pendurava em cada palavra que ela dizia, desesperado para aprender qualquer coisa que o ajudasse a se segurar na vassoura mais tarde, mas todos os outros ficaram muito felizes quando a confer\\u00eancia de Hermione foi interrompida pela chegada do correio. Harry n\\u00e3o recebera nenhuma carta desde o bilh\",\n          \"Qual \\u00e9 o endere\\u00e7o completo de Harry Potter conforme indicado na carta que ele recebeu, incluindo o nome do destinat\\u00e1rio, a localiza\\u00e7\\u00e3o espec\\u00edfica da casa e o c\\u00f3digo postal?\",\n          \"Qual \\u00e9 o nome do bar que Hagrid descreve como um lugar famoso, mas que Harry percebe ser muito escuro e miser\\u00e1vel quando o visitam pela primeira vez?\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"contexto\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 140,\n        \"samples\": [\n          \"A Profa. Minerva, num robe de l\\u00e3 escocesa e rede no cabelo, segurava Malfoy pela orelha. - Est\\u00e1 detido - gritou. - E s\\u00e3o vinte pontos a menos para Sonserina. Perambulando no meio da noite, como voc\\u00ea se atreve...\",\n          \"Ele perdeu a anima\\u00e7\\u00e3o mais depressa do que quando mergulhara. A Profa. Minerva vinha correndo em dire\\u00e7\\u00e3o \\u00e0 turma. Ele se levantou tremendo. - Nunca... em todo o tempo que estou em Hogwarts... A Profa. Minerva quase perdeu a fala de espanto e seus \\u00f3culos cintilavam sem parar, \\\"... como \\u00e9 que voc\\u00ea se atreve... podia ter partido o pesco\\u00e7o...\\\" .\",\n          \"Um rapaz p\\u00e1lido adiantou-se, muito nervoso. Um olho tr\\u00eamulo. - Prof. Quirrell! - disse Hagrid. - Harry, o Prof. Quirrell vai ser um dos seus professores em Hogwarts. - P-P-Potter - gaguejou o Prof. Quirrell, apertando a m\\u00e3o de Harry -, n-n-em sei d-d-dizer que p-p-p-prazer enorme \\u00e9 c-c-conhec\\u00ea-lo. - Que tipo de m\\u00e1gica o senhor ensina, Prof. Quirrell? - D-d-defesa C-c-ontra as Art-t-tes das T-t-trevas - murmurou o Prof. Quirrell, como se preferisse n\\u00e3o pensar no assunto.\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"resposta\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 140,\n        \"samples\": [\n          \"Est\\u00e1 detido - gritou. - E s\\u00e3o vinte pontos a menos para Sonserina.\",\n          \"A Profa. Minerva quase perdeu a fala de espanto e seus \\u00f3culos cintilavam sem parar, \\\"... como \\u00e9 que voc\\u00ea se atreve... podia ter partido o pesco\\u00e7o...\\\"\",\n          \"Prof. Quirrell ensina Defesa Contra as Artes das Trevas.\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"chunk\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 210,\n        \"samples\": [\n          \": Um guia de autoprote\\u00e7\\u00e3o de Quintino Trimble. Outros Equipamentos 1 varinha m\\u00e1gica 1 caldeir\\u00e3o (estanho, tamanho padr\\u00e3o 2) 1 conjunto de frascos 1 telesc\\u00f3pio 1 balan\\u00e7a de lat\\u00e3o Os alunos podem ainda trazer uma coruja OU um gato OU um sapo. LEMBRAMOS AOS PAIS QUE OS ALUNOS DO PRIMEIRO ANO N\\u00c3O PODEM USAR VASSOURAS PESSOAIS. - Podemos comprar tudo isso em Londres? - perguntou-se Harry em voz alta. - Se voc\\u00ea souber aonde ir - respondeu Hagrid. Harry nunca estivera em Londres antes. Hagrid, embora parecesse saber aonde ia, obviamente n\\u00e3o estava acostumado a chegar l\\u00e1 pelos meios comuns. Ficou entalado na roleta do metr\\u00f4 e queixou-se em voz alta que os assentos eram demasiado pequenos e os trens demasiado lentos. - N\\u00e3o sei como os trouxas conseguem se arranjar sem m\\u00e1gica - disse, quando subiam uma escada rolante gasta que levava a uma rua movimentada com lojas dos dois lados. Hagrid era t\\u00e3o grande que abria caminho pela multid\\u00e3o sem esfor\\u00e7o, Harry s\\u00f3 precisava segui-lo de perto. Passaram por livrarias e lojas de m\\u00fasica, lanchonetes e cinemas, mas nenhuma loja parecia vender varinhas m\\u00e1gicas. Aquela era apenas uma rua comum cheia de gente comum. Seria realmente poss\\u00edvel que houvesse montes de ouro dos bruxos enterrados quil\\u00f4metros abaixo dali? Haveria realmente lojas que vendessem livros de feiti\\u00e7os e vassouras? N\\u00e3o seria talvez uma grande pe\\u00e7a que os Dursley tinham pregado? Se Harry n\\u00e3o soubesse que os Dursley n\\u00e3o tinham senso de humor, poderia ter tirado uma dessas conclus\\u00f5es; mas, por alguma raz\\u00e3o, embora tudo que Hagrid tivesse dito at\\u00e9 ali fosse inacredit\\u00e1vel, Harry n\\u00e3o podia deixar de confiar nele. - \\u00c9 aqui - disse Hagrid parando. - O Caldeir\\u00e3o Furado. \\u00c9 um lugar famoso. Era um barzinho sujo. Se Hagrid n\\u00e3o o tivesse apontado, Harry nem teria reparado que existia. As pessoas que passavam apressadas nem olhavam para aquele lado. Os olhos delas corriam da grande livraria a um lado \\u00e0 loja de discos no outro como se nem conseguissem ver O Caldeir\\u00e3o Furado. Na verdade Harry teve a sensa\\u00e7\\u00e3o muito estranha de que somente ele e Hagrid eram capazes de v\\u00ea- lo. Antes que pudesse comentar isto, Hagrid o empurrou para dentro. Para um lugar famoso, o Caldeir\\u00e3o era muito escuro e miser\\u00e1vel. Havia umas velhas sentadas a um canto, bebendo pequenos c\\u00e1lices de xerez. Uma delas fumava um longo cachimbo. Um homenzinho de cartola conversava com o velho dono do bar, que era bem careca e parecia uma noz viscosa. O zum-zum das conversas parou quando eles entraram. Todos pareciam conhecer Hagrid; acenaram e sorriram para ele, Tom apanhou um copo, perguntando: - O de sempre, Hagrid? - N\\u00e3o posso, Tom, estou a servi\\u00e7o de Hogwarts - disse Hagrid, dando uma palmada com a manzorra no ombro de Harry, o que fez os joelhos do garoto dobrarem. - Meu Deus! - exclamou Tom, fitando Harry. - \\u00c9... ser\\u00e1 poss\\u00edvel? O Caldeir\\u00e3o Furado repentinamente parou e fez-se um sil\\u00eancio total. - Valha-me Deus - murmurou o velho Tom. - Harry Potter... que honra. E saiu correndo de tr\\u00e1s do balc\\u00e3o, precipitou-se para Harry e agarrou suas m\\u00e3os, as l\\u00e1grimas nos olhos. - Seja bem-vindo, Sr. Potter, seja bem-vindo. Harry n\\u00e3o sabia o que dizer. Todos tinham os olhos nele. A velha com o cachimbo puxava o f\",\n          \"a ele n\\u00e3o tinham mudado nem um pingo. Isto n\\u00e3o o preocupou. Parecia-lhe que sua vida voltaria ao normal no pr\\u00f3ximo ano, ou t\\u00e3o normal quanto ela poderia ser em Hogwarts. Foi a melhor noite da vida de Harry, melhor do que a vit\\u00f3ria no quadribol ou a ceia de Natal ou o encontro com o trasgo... jamais esqueceria esta noite. Harry quase esquecera que os resultados dos exames ainda estavam por vir, mas eles n\\u00e3o deixaram de vir. Para sua grande surpresa, tanto ele quanto Rony passaram com boas notas; Hermione, \\u00e9 claro, foi a melhor do ano. At\\u00e9 Neville passou raspando, sua boa nota em Herbologia compensou a p\\u00e9ssima nota em Po\\u00e7\\u00f5es. Tinham tido esperan\\u00e7as de que Goyle, que era quase t\\u00e3o burro quanto era mau, fosse expulso, mas ele tamb\\u00e9m passou. Foi uma pena, mas, como disse Rony, n\\u00e3o se podia ter tudo na vida. E, de repente, seus guarda-roupas ficaram vazios, os mal\\u00f5es arrumados, o sapo de Neville foi encontrado escondido em um canto do banheiro; as notas foram entregues a todos os alunos, com o aviso de que n\\u00e3o fizessem bruxarias durante as f\\u00e9rias (\\\"Eu sempre tenho a esperan\\u00e7a de que eles se esque\\u00e7am de entregar as notas e o aviso\\\" - lamentou Fred Weasley); Hagrid estava a postos para lev\\u00e1-los \\u00e0 flotilha de barcos que fazia a travessia do lago; e, no momento seguinte, estavam embarcando no Expresso de Hogwarts; conversando e rindo \\u00e0 medida que os campos se tornavam mais verdes e mais cuidados; comendo feij\\u00f5ezinhos de todos os sabores enquanto atravessavam as cidades dos trouxas; trocando as vestes de bruxos pelos blus\\u00f5es e palet\\u00f3s; parando na plataforma nove e meia na esta\\u00e7\\u00e3o de King\\\"s Cross. Levou um bom tempo para todos desembarcarem na plataforma. Um guarda muito velho estava postado na sa\\u00edda e os deixava passar em grupos de dois e tr\\u00eas para n\\u00e3o chamarem aten\\u00e7\\u00e3o ao irromper todos ao mesmo tempo por uma parede s\\u00f3lida, assustando os trouxas. - Voc\\u00eas precisam vir passar uns dias conosco - disse Rony. - Os dois. Vou mandar uma coruja para voc\\u00eas. - Obrigado - disse Harry. - Preciso ter alguma coisa por que esperar. As pessoas passavam empurrando-os ao se dirigirem para a sa\\u00edda para o mundo dos trouxas. Alguns gritavam: - Tchau, Harry! - Nos vemos por a\\u00ed, Potter! - Continua famoso - comentou Rony, sorrindo para o amigo. - N\\u00e3o aonde eu vou, posso lhe garantir. Ele, Rony e Hermione passaram juntos pelo port\\u00e3o. - Olha l\\u00e1 ele, mam\\u00e3e, olha l\\u00e1 ele, olha! Era Gina Weasley, a irm\\u00e3zinha de Rony, mas n\\u00e3o apontava para Rony. - Harry Potter! - gritou com a vozinha fina. - Olhe, mam\\u00e3e! Estou vendo... - Fique quieta, Gina, \\u00e9 falta de educa\\u00e7\\u00e3o apontar. A Sra. Weasley sorriu para eles. - Muito trabalho este ano? - Muito - respondeu Harry. - Obrigado pelas barrinhas de chocolate e pela su\\u00e9ter, Sra. Weasley. - Ah, de nada, querido. - Est\\u00e1 pronto? Era tio V\\u00e1lter, ainda com a cara vermelhona, ainda bigodudo, ainda parecendo furioso com a aud\\u00e1cia de Harry de andar carregando uma coruja numa gaiola por uma esta\\u00e7\\u00e3o cheia de gente normal. Atr\\u00e1s dele, achavam-se tia Pet\\u00fania e Duda, parecendo aterrorizados s\\u00f3 de olhar para Harry. - Voc\\u00eas devem ser a fam\\u00edlia de Harry! - falou a Sra. Weasley. - Por assim dizer - respondeu tio V\\u00e1lter. - Ande logo, menino, n\\u00e3o temos o dia inteiro. - E se afastou. Harry ainda demorou para trocar uma \\u00faltima palavrinha com Rony e Hermione.\",\n          \"Qual foi a rea\\u00e7\\u00e3o de tio V\\u00e1lter quando Harry mencionou ter sonhado com uma motocicleta que voava?\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"label\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0,\n        \"min\": -1,\n        \"max\": 1,\n        \"num_unique_values\": 2,\n        \"samples\": [\n          -1,\n          1\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"dataset\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 2,\n        \"samples\": [\n          \"test\",\n          \"train\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"pergunta_embedding\",\n      \"properties\": {\n        \"dtype\": \"object\",\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"chunk_embedding\",\n      \"properties\": {\n        \"dtype\": \"object\",\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"cosine_similarity\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.033062434045649396,\n        \"min\": 0.7651159604362697,\n        \"max\": 0.9401074237412784,\n        \"num_unique_values\": 420,\n        \"samples\": [\n          0.8193018014909341,\n          0.8882798140457474\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"pergunta_embedding_custom\",\n      \"properties\": {\n        \"dtype\": \"object\",\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"chunk_embedding_custom\",\n      \"properties\": {\n        \"dtype\": \"object\",\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"cosine_similarity_custom\",\n      \"properties\": {\n        \"dtype\": \"float32\",\n        \"num_unique_values\": 420,\n        \"samples\": [\n          -0.041865769773721695,\n          0.17222636938095093\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_id = pd.DataFrame(list(corpus.items()), columns=['id', 'conteudo'])\n",
        "df_id[\"pergunta\"] = df_id[\"id\"].map(topics)\n",
        "df_id.head(2)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 112
        },
        "id": "vB9BSLqb4--2",
        "outputId": "52549ae3-a7bb-46ad-d8c7-3a506cbe0656"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   id                                           conteudo  \\\n",
              "0   0  - CAPÍTULO UM - O menino que sobreviveu O Sr. ...   \n",
              "1   1  ante o caminho para a cidade ele não pensou em...   \n",
              "\n",
              "                                            pergunta  \n",
              "0  Qual é o segredo que o Sr. e a Sra. Dursley te...  \n",
              "1  Qual era a cor da capa que o Sr. Dursley viu u...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-a917bb20-e90e-49af-b3ee-f79cf05c8fdf\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>conteudo</th>\n",
              "      <th>pergunta</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>- CAPÍTULO UM - O menino que sobreviveu O Sr. ...</td>\n",
              "      <td>Qual é o segredo que o Sr. e a Sra. Dursley te...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>ante o caminho para a cidade ele não pensou em...</td>\n",
              "      <td>Qual era a cor da capa que o Sr. Dursley viu u...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-a917bb20-e90e-49af-b3ee-f79cf05c8fdf')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-a917bb20-e90e-49af-b3ee-f79cf05c8fdf button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-a917bb20-e90e-49af-b3ee-f79cf05c8fdf');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-846245dc-70b0-464c-ac73-ed32f4c9a092\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-846245dc-70b0-464c-ac73-ed32f4c9a092')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-846245dc-70b0-464c-ac73-ed32f4c9a092 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "df_id",
              "summary": "{\n  \"name\": \"df_id\",\n  \"rows\": 140,\n  \"fields\": [\n    {\n      \"column\": \"id\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 40,\n        \"min\": 0,\n        \"max\": 139,\n        \"num_unique_values\": 140,\n        \"samples\": [\n          108,\n          67,\n          31\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"conteudo\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 140,\n        \"samples\": [\n          \" Harry facilitou muito o transporte. - Estamos quase l\\u00e1! - Harry ofegou quando chegaram ao corredor sob a torre mais alta. Ent\\u00e3o um movimento brusco \\u00e0 frente deles quase fez com que deixassem cair o caixote. Esquecendo que j\\u00e1 estavam invis\\u00edveis, encolheram-se nas sombras, espiando os contornos escuros de duas pessoas que se debatiam a uns tr\\u00eas metros. Uma l\\u00e2mpada se acendeu. A Profa. Minerva, num robe de l\\u00e3 escocesa e rede no cabelo, segurava Malfoy pela orelha. - Est\\u00e1 detido - gritou. - E s\\u00e3o vinte pontos a menos para Sonserina. Perambulando no meio da noite, como voc\\u00ea se atreve... - A senhora n\\u00e3o compreende, professora, Harry Potter est\\u00e1 vindo a\\u00ed; vem trazendo um drag\\u00e3o. - Que absurdo! Como voc\\u00ea se atreve a contar tais mentiras? Vamos: vou conversar com o Prof. Snape sobre voc\\u00ea, Malfoy! A \\u00edngreme escada em espiral at\\u00e9 o alto da torre pareceu a coisa mais f\\u00e1cil do mundo depois disto. Somente quando sa\\u00edram para o ar frio da noite foi que se livraram da capa da invisibilidade, felizes de poderem respirar direito outra vez. Hermione dan\\u00e7ou uma esp\\u00e9cie de jiga escocesa. - Malfoy vai ficar detido! Eu seria capaz de cantar. - N\\u00e3o cante - aconselhou Harry. Rindo de Malfoy, eles esperaram, enquanto Norberto se debatia dentro do  caixote. Passados uns dez minutos, quatro vassouras surgiram da escurid\\u00e3o mergulhando em dire\\u00e7\\u00e3o \\u00e0 torre. Os amigos de Carlinhos formavam um grupo animado. Mostraram a Harry e a Hermione os arreios que tinham trazido de modo a poder suspender Norberto entre eles. Todos ajudaram a prender Norberto muito bem nos arreios e ent\\u00e3o Harry e Hermione apertaram as m\\u00e3os de todos e lhes agradeceram muito. Finalmente Norberto estava indo... indo... e finalmente se foi. Eles desceram a escada espiral sem fazer barulho, os cora\\u00e7\\u00f5es leves como as m\\u00e3os, agora que Norberto fora tirado delas. Nada de drag\\u00e3o - Malfoy detido - o que poderia estragar essa felicidade? A resposta \\u00e0 sua pergunta estava esperando ao p\\u00e9 da escada. Quando chegaram ao corredor, a cara de Filch assombrou-os, emergindo da escurid\\u00e3o. - Ora, ora, ora - sussurrou -, estamos encrencados. Tinham deixado a capa da invisibilidade no alto da torre.  - CAP\\u00cdTULO QUINZE - A Floresta Proibida As coisas n\\u00e3o poderiam estar piores. Filch levou-os \\u00e0 sala da Profa. Minerva no primeiro andar, onde eles ficaram sentados esperando, sem trocar uma palavra entre si. Hermione tremia. Desculpas, \\u00e1libis e justificativas fant\\u00e1sticas substitu\\u00edam-se umas \\u00e0s outras na cabe\\u00e7a de Harry, cada qual mais capenga do que a anterior. Ele n\\u00e3o conseguia ver como iam se livrar desta encrenca. Estavam encurralados. Como podiam ter sido burros a ponto de se esquecerem da capa? N\\u00e3o havia nenhuma raz\\u00e3o no mundo para a Profa. Minerva aceitar que estivessem fora da cama, esgueirando- se pela escola a altas horas da noite, e muito menos que estivessem na alta torre de astronomia, que era proibida aos alunos a n\\u00e3o ser durante as aulas. Some-se a isso Norberto e a capa da invisibilidade e seria melhor come\\u00e7arem a fazer as malas. Harry achou que as coisas n\\u00e3o poderiam ficar piores. Estava enganado. Quando a Profa. Minerva apareceu, vinha trazendo Neville. - Harry! - exclamou ele, no instante em que viu os outros dois. - Eu estava tentando encontrar voc\\u00eas para avisar que ouvi Malfoy dizer que ia pegar voc\\u00eas, disse que voc\\u00eas tinham um drag... Harry sacudiu com for\\u00e7a a cabe\\u00e7a para fazer Neville calar a boca\",\n          \" de alguma maneira sabia o que fazer. Curvou-se para a frente, segurou a vassoura com firmeza com as duas m\\u00e3os e ela disparou na dire\\u00e7\\u00e3o de Draco como uma lan\\u00e7a. Draco s\\u00f3 conseguiu escapar por um triz; Harry fez uma curva fechada e manteve a vassoura firme. Algumas pessoas no ch\\u00e3o aplaudiram. - Aqui n\\u00e3o tem Crabbe nem Goyle para salvarem sua pele, Draco - berrou Harry. O mesmo pensamento parecia ter ocorrido a Draco. - Apanhe se puder, ent\\u00e3o! - gritou, atirou a bolinha de cristal no ar e voltou para o ch\\u00e3o. Harry viu, como se fosse em c\\u00e2mara lenta, a bolinha subir no ar e come\\u00e7ar a cair. Ele se curvou para a frente e apontou o cabo da vassoura para baixo - no instante seguinte estava ganhando velocidade num mergulho quase vertical, apostando corrida com a bolinha - o vento assobiava em suas orelhas, misturado aos gritos das pessoas que olhavam - ele esticou a m\\u00e3o - a uns trinta cent\\u00edmetros do solo agarrou-a, bem em tempo de levar a vassoura \\u00e0 posi\\u00e7\\u00e3o vertical, e caiu suavemente na grama com o Lembrol salvo e seguro na m\\u00e3o. - HARRY POTTER! Ele perdeu a anima\\u00e7\\u00e3o mais depressa do que quando mergulhara. A Profa. Minerva vinha correndo em dire\\u00e7\\u00e3o \\u00e0 turma. Ele se levantou tremendo. - Nunca... em todo o tempo que estou em Hogwarts... A Profa. Minerva quase perdeu a fala de espanto e seus \\u00f3culos cintilavam sem parar, \\\"... como \\u00e9 que voc\\u00ea se atreve... podia ter partido o pesco\\u00e7o...\\\" . - N\\u00e3o foi culpa dele, professora... - Calada, Srta. Patil... - Mas Draco... - Chega, Sr. Weasley. Potter, me acompanhe, agora. Harry viu as caras vitoriosas de Draco, Crabbe e Goyle ao sair acompanhando, espantado, a Profa. Minerva, que seguiu para o castelo. Ia ser expulso, sabia. Queria dizer alguma coisa para se defender, mas parecia ter acontecido alguma coisa com a sua voz. A Profa. Minerva caminhava decidida, sem nem olhar para tr\\u00e1s; ele tinha que correr para acompanhar seu passo. Agora se enrascara. N\\u00e3o tinha durado nem duas semanas. Estaria fazendo as malas dali a dez minutos. Que iriam dizer os Dursley quando ele aparecesse \\u00e0 porta da casa? Subiram os degraus da entrada, subiram a escadaria de m\\u00e1rmore, e a Profa. Minerva continuava a n\\u00e3o dizer nada. Escancarava portas e marchava pelos corredores com Harry trotando infeliz atr\\u00e1s dela. Talvez ela o levasse a Dumbledore. Pensou em Hagrid, aluno expulso a quem tinham permitido continuar na escola como guarda-ca\\u00e7a. Talvez virasse assistente de Hagrid. Seu  est\\u00f4mago revirava s\\u00f3 de pensar, observando Rony e os outros se tornarem bruxos enquanto ele andava pela propriedade carregando a bolsa de Hagrid. A Profa. Minerva parou \\u00e0 porta de uma sala de aula. Abriu a porta e meteu a cabe\\u00e7a para dentro. - Com licen\\u00e7a, Prof. Flitwick, posso pedir o Wood emprestado por um instante? Wood?, pensou Harry, intrigado; Wood seria alguma coisa que ela ia usar para castig\\u00e1-lo? Mas Wood afinal era uma pessoa, um menino forte do quinto ano, que saiu da sala de Flitwick parecendo confuso. - Voc\\u00eas dois me sigam - disse a Profa. Minerva, e continuaram todos pelo corredor, Wood examinando Harry com curiosidade. - Entrem. A Profa. Minerva indicou uma sala de aula que estava vazia exceto por Pirra\\u00e7a, que se ocupava em escrever palavr\\u00f5es no quadro-negro. - Fora, Pirra\\u00e7a! - ordenou ela. Pirra\\u00e7a atirou o giz\",\n          \"umo sem se dar conta de que o cachimbo apagara. Hagrid sorria radiante. Logo houve um grande arrastar de cadeiras e no momento seguinte Harry se viu apertando as m\\u00e3os de todos n\\\"O Caldeir\\u00e3o Furado. - D\\u00f3ris Crockford, Sr. Potter, n\\u00e3o acredito que finalmente posso conhec\\u00ea-lo. - Estou t\\u00e3o orgulhosa, Sr. Potter, t\\u00e3o orgulhosa. - Sempre quis apertar sua m\\u00e3o. Estou nas nuvens. - Encantado, Sr. Potter, nem sei lhe dizer o quanto, Diggle \\u00e9 o meu nome, D\\u00e9dalo Diggle. - J\\u00e1 vi o senhor antes! - disse Harry, e a cartola de Diggle caiu de tanta excita\\u00e7\\u00e3o. - O senhor se curvou para mim uma vez numa loja. - Ele se lembra! - exclamou D\\u00e9dalo Diggle, olhando todos \\u00e0 volta. - Voc\\u00eas ouviram isso? Ele se lembra de mim! Harry apertou muitas m\\u00e3os. D\\u00f3ris Crockford n\\u00e3o parava de voltar para um novo aperto.  Um rapaz p\\u00e1lido adiantou-se, muito nervoso. Um olho tr\\u00eamulo. - Prof. Quirrell! - disse Hagrid. - Harry, o Prof. Quirrell vai ser um dos seus professores em Hogwarts. - P-P-Potter - gaguejou o Prof. Quirrell, apertando a m\\u00e3o de Harry -, n-n-em sei d-d-dizer que p-p-p-prazer enorme \\u00e9 c-c-conhec\\u00ea-lo. - Que tipo de m\\u00e1gica o senhor ensina, Prof. Quirrell? - D-d-defesa C-c-ontra as Art-t-tes das T-t-trevas - murmurou o Prof. Quirrell, como se preferisse n\\u00e3o pensar no assunto. - N-n-n\\u00e3o que voc\\u00ea p-p- precise, hein, Potter? - Ele riu nervoso. - V-v-voc\\u00ea veio c-c-comprar o material, suponho? Tenho que c-c-comprar um livro n-n-novo sobre vampiros. - Parecia aterrorizado s\\u00f3 de pensar. Mas os outros n\\u00e3o queriam deixar o Prof. Quirrell ficar com Harry s\\u00f3 para ele. Levou bem uns dez minutos para o menino se livrar de todos. Finalmente, Hagrid conseguiu se fazer ouvir naquela balb\\u00fardia. - Precisamos nos apressar. Temos muitas compras a fazer. Vamos, Harry. D\\u00f3ris Crockford apertou a m\\u00e3o de Harry uma \\u00faltima vez e eles passaram pelo bar e sa\\u00edram num pequeno p\\u00e1tio murado, onde n\\u00e3o havia nada exceto uma lata de lixo e um pouco de mato. Hagrid sorriu para Harry. - Eu lhe falei, n\\u00e3o foi? Falei que voc\\u00ea era famoso. At\\u00e9 o professor Quirrell ficou tremendo de emo\\u00e7\\u00e3o de o conhecer, mas, em geral, ele est\\u00e1 sempre tremendo. - Ele \\u00e9 sempre t\\u00e3o nervoso? - Ah, \\u00e9. Coitado. Uma cabe\\u00e7a brilhante. Foi bem enquanto estudou em livros, mas quando tirou um ano para aprender na pr\\u00e1tica... Dizem que encontrou vampiros na Floresta Negra e teve um problema feio com uma feiticeira, nunca mais foi o mesmo. Tem pavor dos alunos, tem pavor da mat\\u00e9ria que ensina, agora, cad\\u00ea o meu guarda-chuva? Vampiros? Feiticeiras? A cabe\\u00e7a de Harry estava girando. Entrementes, Hagrid contava tijolos na parede por cima da lata de lixo. - Tr\\u00eas para cima... dois para o lado... - murmurou. - Certo, chegue para tr\\u00e1s, Harry. Ele bateu na parede tr\\u00eas vezes com a ponta do guarda-chuva. E o tijolo que tocou estremeceu, ent\\u00e3o torceu-se. No meio apareceu um buraquinho, que se foi alargando cada vez mais. Um segundo depois se viram diante de um arco bastante grande at\\u00e9 para Hagrid, um arco que abria para uma rua de pedras irregulares, serpeava e desaparecia de vista. - Bem-vindo - disse Hagrid - ao Beco Diagonal. Ele riu do espanto de Harry. A\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"pergunta\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 140,\n        \"samples\": [\n          \"Qual foi a puni\\u00e7\\u00e3o imposta pela Profa. Minerva a Malfoy por estar perambulando no meio da noite e qual foi a consequ\\u00eancia para a casa dele?\",\n          \"Qual foi a rea\\u00e7\\u00e3o da Professora Minerva ao ver a manobra arriscada de Harry com a vassoura e o que ela disse a ele?\",\n          \"Qual \\u00e9 o nome completo do professor que Harry conhece no Caldeir\\u00e3o Furado e que mat\\u00e9ria ele ensina em Hogwarts?\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 41
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# pergunta:id\n",
        "map_dict = {k:v for k,v in zip(df_id[\"pergunta\"], df_id[\"id\"])}\n",
        "\n",
        "df_custom['id'] = df_custom.pergunta.apply(lambda x: map_dict[x] if x in map_dict.keys() else -1)\n",
        "\n",
        "df_ada002_custom = df_custom[(df_custom.id != -1) & (df_custom.label == 1)]\n",
        "print(len(df_ada002_custom))\n",
        "df_ada002_custom.head(2)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 200
        },
        "id": "SSryVVMA5Ikd",
        "outputId": "68904451-a8fc-4743-a8b5-3e2bafb127b5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "140\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                            pergunta  \\\n",
              "0  Qual é o segredo que o Sr. e a Sra. Dursley te...   \n",
              "1  Qual era a cor da capa que o Sr. Dursley viu u...   \n",
              "\n",
              "                                            contexto  \\\n",
              "0  Os Dursley tinham tudo que queriam, mas tinham...   \n",
              "1  O Sr. Dursley não tolerava gente que andava co...   \n",
              "\n",
              "                   resposta  \\\n",
              "0   a existência dos Potter   \n",
              "1  uma capa verde-esmeralda   \n",
              "\n",
              "                                               chunk  label dataset  \\\n",
              "0  - CAPÍTULO UM - O menino que sobreviveu O Sr. ...      1   train   \n",
              "1  ante o caminho para a cidade ele não pensou em...      1   train   \n",
              "\n",
              "                                  pergunta_embedding  \\\n",
              "0  [0.022689364850521088, 0.0038326631765812635, ...   \n",
              "1  [0.005216388031840324, 0.0008732206770218909, ...   \n",
              "\n",
              "                                     chunk_embedding  cosine_similarity  \\\n",
              "0  [0.014419169165194035, 0.011491919867694378, -...           0.873794   \n",
              "1  [-0.002306348877027631, -0.0014758666511625051...           0.873999   \n",
              "\n",
              "                           pergunta_embedding_custom  \\\n",
              "0  [0.17453432, -0.60518175, -0.7326583, 0.358211...   \n",
              "1  [-0.62348986, -0.43539473, -0.32246643, -0.106...   \n",
              "\n",
              "                              chunk_embedding_custom  \\\n",
              "0  [0.024289679, -0.73309463, -0.37434945, 0.1869...   \n",
              "1  [-0.28205645, -0.14895901, 0.066049635, 0.5515...   \n",
              "\n",
              "   cosine_similarity_custom  id  \n",
              "0                  0.398477   0  \n",
              "1                  0.479307   1  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-535d18bd-5b55-4bd5-b202-1293416b4f04\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>pergunta</th>\n",
              "      <th>contexto</th>\n",
              "      <th>resposta</th>\n",
              "      <th>chunk</th>\n",
              "      <th>label</th>\n",
              "      <th>dataset</th>\n",
              "      <th>pergunta_embedding</th>\n",
              "      <th>chunk_embedding</th>\n",
              "      <th>cosine_similarity</th>\n",
              "      <th>pergunta_embedding_custom</th>\n",
              "      <th>chunk_embedding_custom</th>\n",
              "      <th>cosine_similarity_custom</th>\n",
              "      <th>id</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Qual é o segredo que o Sr. e a Sra. Dursley te...</td>\n",
              "      <td>Os Dursley tinham tudo que queriam, mas tinham...</td>\n",
              "      <td>a existência dos Potter</td>\n",
              "      <td>- CAPÍTULO UM - O menino que sobreviveu O Sr. ...</td>\n",
              "      <td>1</td>\n",
              "      <td>train</td>\n",
              "      <td>[0.022689364850521088, 0.0038326631765812635, ...</td>\n",
              "      <td>[0.014419169165194035, 0.011491919867694378, -...</td>\n",
              "      <td>0.873794</td>\n",
              "      <td>[0.17453432, -0.60518175, -0.7326583, 0.358211...</td>\n",
              "      <td>[0.024289679, -0.73309463, -0.37434945, 0.1869...</td>\n",
              "      <td>0.398477</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Qual era a cor da capa que o Sr. Dursley viu u...</td>\n",
              "      <td>O Sr. Dursley não tolerava gente que andava co...</td>\n",
              "      <td>uma capa verde-esmeralda</td>\n",
              "      <td>ante o caminho para a cidade ele não pensou em...</td>\n",
              "      <td>1</td>\n",
              "      <td>train</td>\n",
              "      <td>[0.005216388031840324, 0.0008732206770218909, ...</td>\n",
              "      <td>[-0.002306348877027631, -0.0014758666511625051...</td>\n",
              "      <td>0.873999</td>\n",
              "      <td>[-0.62348986, -0.43539473, -0.32246643, -0.106...</td>\n",
              "      <td>[-0.28205645, -0.14895901, 0.066049635, 0.5515...</td>\n",
              "      <td>0.479307</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-535d18bd-5b55-4bd5-b202-1293416b4f04')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-535d18bd-5b55-4bd5-b202-1293416b4f04 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-535d18bd-5b55-4bd5-b202-1293416b4f04');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-80291b46-ed48-4ee8-a6e4-5f439d69baba\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-80291b46-ed48-4ee8-a6e4-5f439d69baba')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-80291b46-ed48-4ee8-a6e4-5f439d69baba button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "df_ada002_custom",
              "summary": "{\n  \"name\": \"df_ada002_custom\",\n  \"rows\": 140,\n  \"fields\": [\n    {\n      \"column\": \"pergunta\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 140,\n        \"samples\": [\n          \"Qual foi a puni\\u00e7\\u00e3o imposta pela Profa. Minerva a Malfoy por estar perambulando no meio da noite e qual foi a consequ\\u00eancia para a casa dele?\",\n          \"Qual foi a rea\\u00e7\\u00e3o da Professora Minerva ao ver a manobra arriscada de Harry com a vassoura e o que ela disse a ele?\",\n          \"Qual \\u00e9 o nome completo do professor que Harry conhece no Caldeir\\u00e3o Furado e que mat\\u00e9ria ele ensina em Hogwarts?\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"contexto\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 140,\n        \"samples\": [\n          \"A Profa. Minerva, num robe de l\\u00e3 escocesa e rede no cabelo, segurava Malfoy pela orelha. - Est\\u00e1 detido - gritou. - E s\\u00e3o vinte pontos a menos para Sonserina. Perambulando no meio da noite, como voc\\u00ea se atreve...\",\n          \"Ele perdeu a anima\\u00e7\\u00e3o mais depressa do que quando mergulhara. A Profa. Minerva vinha correndo em dire\\u00e7\\u00e3o \\u00e0 turma. Ele se levantou tremendo. - Nunca... em todo o tempo que estou em Hogwarts... A Profa. Minerva quase perdeu a fala de espanto e seus \\u00f3culos cintilavam sem parar, \\\"... como \\u00e9 que voc\\u00ea se atreve... podia ter partido o pesco\\u00e7o...\\\" .\",\n          \"Um rapaz p\\u00e1lido adiantou-se, muito nervoso. Um olho tr\\u00eamulo. - Prof. Quirrell! - disse Hagrid. - Harry, o Prof. Quirrell vai ser um dos seus professores em Hogwarts. - P-P-Potter - gaguejou o Prof. Quirrell, apertando a m\\u00e3o de Harry -, n-n-em sei d-d-dizer que p-p-p-prazer enorme \\u00e9 c-c-conhec\\u00ea-lo. - Que tipo de m\\u00e1gica o senhor ensina, Prof. Quirrell? - D-d-defesa C-c-ontra as Art-t-tes das T-t-trevas - murmurou o Prof. Quirrell, como se preferisse n\\u00e3o pensar no assunto.\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"resposta\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 140,\n        \"samples\": [\n          \"Est\\u00e1 detido - gritou. - E s\\u00e3o vinte pontos a menos para Sonserina.\",\n          \"A Profa. Minerva quase perdeu a fala de espanto e seus \\u00f3culos cintilavam sem parar, \\\"... como \\u00e9 que voc\\u00ea se atreve... podia ter partido o pesco\\u00e7o...\\\"\",\n          \"Prof. Quirrell ensina Defesa Contra as Artes das Trevas.\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"chunk\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 140,\n        \"samples\": [\n          \"Harry facilitou muito o transporte. - Estamos quase l\\u00e1! - Harry ofegou quando chegaram ao corredor sob a torre mais alta. Ent\\u00e3o um movimento brusco \\u00e0 frente deles quase fez com que deixassem cair o caixote. Esquecendo que j\\u00e1 estavam invis\\u00edveis, encolheram-se nas sombras, espiando os contornos escuros de duas pessoas que se debatiam a uns tr\\u00eas metros. Uma l\\u00e2mpada se acendeu. A Profa. Minerva, num robe de l\\u00e3 escocesa e rede no cabelo, segurava Malfoy pela orelha. - Est\\u00e1 detido - gritou. - E s\\u00e3o vinte pontos a menos para Sonserina. Perambulando no meio da noite, como voc\\u00ea se atreve... - A senhora n\\u00e3o compreende, professora, Harry Potter est\\u00e1 vindo a\\u00ed; vem trazendo um drag\\u00e3o. - Que absurdo! Como voc\\u00ea se atreve a contar tais mentiras? Vamos: vou conversar com o Prof. Snape sobre voc\\u00ea, Malfoy! A \\u00edngreme escada em espiral at\\u00e9 o alto da torre pareceu a coisa mais f\\u00e1cil do mundo depois disto. Somente quando sa\\u00edram para o ar frio da noite foi que se livraram da capa da invisibilidade, felizes de poderem respirar direito outra vez. Hermione dan\\u00e7ou uma esp\\u00e9cie de jiga escocesa. - Malfoy vai ficar detido! Eu seria capaz de cantar. - N\\u00e3o cante - aconselhou Harry. Rindo de Malfoy, eles esperaram, enquanto Norberto se debatia dentro do caixote. Passados uns dez minutos, quatro vassouras surgiram da escurid\\u00e3o mergulhando em dire\\u00e7\\u00e3o \\u00e0 torre. Os amigos de Carlinhos formavam um grupo animado. Mostraram a Harry e a Hermione os arreios que tinham trazido de modo a poder suspender Norberto entre eles. Todos ajudaram a prender Norberto muito bem nos arreios e ent\\u00e3o Harry e Hermione apertaram as m\\u00e3os de todos e lhes agradeceram muito. Finalmente Norberto estava indo... indo... e finalmente se foi. Eles desceram a escada espiral sem fazer barulho, os cora\\u00e7\\u00f5es leves como as m\\u00e3os, agora que Norberto fora tirado delas. Nada de drag\\u00e3o - Malfoy detido - o que poderia estragar essa felicidade? A resposta \\u00e0 sua pergunta estava esperando ao p\\u00e9 da escada. Quando chegaram ao corredor, a cara de Filch assombrou-os, emergindo da escurid\\u00e3o. - Ora, ora, ora - sussurrou -, estamos encrencados. Tinham deixado a capa da invisibilidade no alto da torre. - CAP\\u00cdTULO QUINZE - A Floresta Proibida As coisas n\\u00e3o poderiam estar piores. Filch levou-os \\u00e0 sala da Profa. Minerva no primeiro andar, onde eles ficaram sentados esperando, sem trocar uma palavra entre si. Hermione tremia. Desculpas, \\u00e1libis e justificativas fant\\u00e1sticas substitu\\u00edam-se umas \\u00e0s outras na cabe\\u00e7a de Harry, cada qual mais capenga do que a anterior. Ele n\\u00e3o conseguia ver como iam se livrar desta encrenca. Estavam encurralados. Como podiam ter sido burros a ponto de se esquecerem da capa? N\\u00e3o havia nenhuma raz\\u00e3o no mundo para a Profa. Minerva aceitar que estivessem fora da cama, esgueirando- se pela escola a altas horas da noite, e muito menos que estivessem na alta torre de astronomia, que era proibida aos alunos a n\\u00e3o ser durante as aulas. Some-se a isso Norberto e a capa da invisibilidade e seria melhor come\\u00e7arem a fazer as malas. Harry achou que as coisas n\\u00e3o poderiam ficar piores. Estava enganado. Quando a Profa. Minerva apareceu, vinha trazendo Neville. - Harry! - exclamou ele, no instante em que viu os outros dois. - Eu estava tentando encontrar voc\\u00eas para avisar que ouvi Malfoy dizer que ia pegar voc\\u00eas, disse que voc\\u00eas tinham um drag... Harry sacudiu com for\\u00e7a a cabe\\u00e7a para fazer Neville calar a boca\",\n          \"de alguma maneira sabia o que fazer. Curvou-se para a frente, segurou a vassoura com firmeza com as duas m\\u00e3os e ela disparou na dire\\u00e7\\u00e3o de Draco como uma lan\\u00e7a. Draco s\\u00f3 conseguiu escapar por um triz; Harry fez uma curva fechada e manteve a vassoura firme. Algumas pessoas no ch\\u00e3o aplaudiram. - Aqui n\\u00e3o tem Crabbe nem Goyle para salvarem sua pele, Draco - berrou Harry. O mesmo pensamento parecia ter ocorrido a Draco. - Apanhe se puder, ent\\u00e3o! - gritou, atirou a bolinha de cristal no ar e voltou para o ch\\u00e3o. Harry viu, como se fosse em c\\u00e2mara lenta, a bolinha subir no ar e come\\u00e7ar a cair. Ele se curvou para a frente e apontou o cabo da vassoura para baixo - no instante seguinte estava ganhando velocidade num mergulho quase vertical, apostando corrida com a bolinha - o vento assobiava em suas orelhas, misturado aos gritos das pessoas que olhavam - ele esticou a m\\u00e3o - a uns trinta cent\\u00edmetros do solo agarrou-a, bem em tempo de levar a vassoura \\u00e0 posi\\u00e7\\u00e3o vertical, e caiu suavemente na grama com o Lembrol salvo e seguro na m\\u00e3o. - HARRY POTTER! Ele perdeu a anima\\u00e7\\u00e3o mais depressa do que quando mergulhara. A Profa. Minerva vinha correndo em dire\\u00e7\\u00e3o \\u00e0 turma. Ele se levantou tremendo. - Nunca... em todo o tempo que estou em Hogwarts... A Profa. Minerva quase perdeu a fala de espanto e seus \\u00f3culos cintilavam sem parar, \\\"... como \\u00e9 que voc\\u00ea se atreve... podia ter partido o pesco\\u00e7o...\\\" . - N\\u00e3o foi culpa dele, professora... - Calada, Srta. Patil... - Mas Draco... - Chega, Sr. Weasley. Potter, me acompanhe, agora. Harry viu as caras vitoriosas de Draco, Crabbe e Goyle ao sair acompanhando, espantado, a Profa. Minerva, que seguiu para o castelo. Ia ser expulso, sabia. Queria dizer alguma coisa para se defender, mas parecia ter acontecido alguma coisa com a sua voz. A Profa. Minerva caminhava decidida, sem nem olhar para tr\\u00e1s; ele tinha que correr para acompanhar seu passo. Agora se enrascara. N\\u00e3o tinha durado nem duas semanas. Estaria fazendo as malas dali a dez minutos. Que iriam dizer os Dursley quando ele aparecesse \\u00e0 porta da casa? Subiram os degraus da entrada, subiram a escadaria de m\\u00e1rmore, e a Profa. Minerva continuava a n\\u00e3o dizer nada. Escancarava portas e marchava pelos corredores com Harry trotando infeliz atr\\u00e1s dela. Talvez ela o levasse a Dumbledore. Pensou em Hagrid, aluno expulso a quem tinham permitido continuar na escola como guarda-ca\\u00e7a. Talvez virasse assistente de Hagrid. Seu est\\u00f4mago revirava s\\u00f3 de pensar, observando Rony e os outros se tornarem bruxos enquanto ele andava pela propriedade carregando a bolsa de Hagrid. A Profa. Minerva parou \\u00e0 porta de uma sala de aula. Abriu a porta e meteu a cabe\\u00e7a para dentro. - Com licen\\u00e7a, Prof. Flitwick, posso pedir o Wood emprestado por um instante? Wood?, pensou Harry, intrigado; Wood seria alguma coisa que ela ia usar para castig\\u00e1-lo? Mas Wood afinal era uma pessoa, um menino forte do quinto ano, que saiu da sala de Flitwick parecendo confuso. - Voc\\u00eas dois me sigam - disse a Profa. Minerva, e continuaram todos pelo corredor, Wood examinando Harry com curiosidade. - Entrem. A Profa. Minerva indicou uma sala de aula que estava vazia exceto por Pirra\\u00e7a, que se ocupava em escrever palavr\\u00f5es no quadro-negro. - Fora, Pirra\\u00e7a! - ordenou ela. Pirra\\u00e7a atirou o giz\",\n          \"umo sem se dar conta de que o cachimbo apagara. Hagrid sorria radiante. Logo houve um grande arrastar de cadeiras e no momento seguinte Harry se viu apertando as m\\u00e3os de todos n\\\"O Caldeir\\u00e3o Furado. - D\\u00f3ris Crockford, Sr. Potter, n\\u00e3o acredito que finalmente posso conhec\\u00ea-lo. - Estou t\\u00e3o orgulhosa, Sr. Potter, t\\u00e3o orgulhosa. - Sempre quis apertar sua m\\u00e3o. Estou nas nuvens. - Encantado, Sr. Potter, nem sei lhe dizer o quanto, Diggle \\u00e9 o meu nome, D\\u00e9dalo Diggle. - J\\u00e1 vi o senhor antes! - disse Harry, e a cartola de Diggle caiu de tanta excita\\u00e7\\u00e3o. - O senhor se curvou para mim uma vez numa loja. - Ele se lembra! - exclamou D\\u00e9dalo Diggle, olhando todos \\u00e0 volta. - Voc\\u00eas ouviram isso? Ele se lembra de mim! Harry apertou muitas m\\u00e3os. D\\u00f3ris Crockford n\\u00e3o parava de voltar para um novo aperto. Um rapaz p\\u00e1lido adiantou-se, muito nervoso. Um olho tr\\u00eamulo. - Prof. Quirrell! - disse Hagrid. - Harry, o Prof. Quirrell vai ser um dos seus professores em Hogwarts. - P-P-Potter - gaguejou o Prof. Quirrell, apertando a m\\u00e3o de Harry -, n-n-em sei d-d-dizer que p-p-p-prazer enorme \\u00e9 c-c-conhec\\u00ea-lo. - Que tipo de m\\u00e1gica o senhor ensina, Prof. Quirrell? - D-d-defesa C-c-ontra as Art-t-tes das T-t-trevas - murmurou o Prof. Quirrell, como se preferisse n\\u00e3o pensar no assunto. - N-n-n\\u00e3o que voc\\u00ea p-p- precise, hein, Potter? - Ele riu nervoso. - V-v-voc\\u00ea veio c-c-comprar o material, suponho? Tenho que c-c-comprar um livro n-n-novo sobre vampiros. - Parecia aterrorizado s\\u00f3 de pensar. Mas os outros n\\u00e3o queriam deixar o Prof. Quirrell ficar com Harry s\\u00f3 para ele. Levou bem uns dez minutos para o menino se livrar de todos. Finalmente, Hagrid conseguiu se fazer ouvir naquela balb\\u00fardia. - Precisamos nos apressar. Temos muitas compras a fazer. Vamos, Harry. D\\u00f3ris Crockford apertou a m\\u00e3o de Harry uma \\u00faltima vez e eles passaram pelo bar e sa\\u00edram num pequeno p\\u00e1tio murado, onde n\\u00e3o havia nada exceto uma lata de lixo e um pouco de mato. Hagrid sorriu para Harry. - Eu lhe falei, n\\u00e3o foi? Falei que voc\\u00ea era famoso. At\\u00e9 o professor Quirrell ficou tremendo de emo\\u00e7\\u00e3o de o conhecer, mas, em geral, ele est\\u00e1 sempre tremendo. - Ele \\u00e9 sempre t\\u00e3o nervoso? - Ah, \\u00e9. Coitado. Uma cabe\\u00e7a brilhante. Foi bem enquanto estudou em livros, mas quando tirou um ano para aprender na pr\\u00e1tica... Dizem que encontrou vampiros na Floresta Negra e teve um problema feio com uma feiticeira, nunca mais foi o mesmo. Tem pavor dos alunos, tem pavor da mat\\u00e9ria que ensina, agora, cad\\u00ea o meu guarda-chuva? Vampiros? Feiticeiras? A cabe\\u00e7a de Harry estava girando. Entrementes, Hagrid contava tijolos na parede por cima da lata de lixo. - Tr\\u00eas para cima... dois para o lado... - murmurou. - Certo, chegue para tr\\u00e1s, Harry. Ele bateu na parede tr\\u00eas vezes com a ponta do guarda-chuva. E o tijolo que tocou estremeceu, ent\\u00e3o torceu-se. No meio apareceu um buraquinho, que se foi alargando cada vez mais. Um segundo depois se viram diante de um arco bastante grande at\\u00e9 para Hagrid, um arco que abria para uma rua de pedras irregulares, serpeava e desaparecia de vista. - Bem-vindo - disse Hagrid - ao Beco Diagonal. Ele riu do espanto de Harry. A\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"label\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0,\n        \"min\": 1,\n        \"max\": 1,\n        \"num_unique_values\": 1,\n        \"samples\": [\n          1\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"dataset\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 2,\n        \"samples\": [\n          \"test\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"pergunta_embedding\",\n      \"properties\": {\n        \"dtype\": \"object\",\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"chunk_embedding\",\n      \"properties\": {\n        \"dtype\": \"object\",\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"cosine_similarity\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.020574514769817462,\n        \"min\": 0.7975285516914384,\n        \"max\": 0.9104405194845736,\n        \"num_unique_values\": 140,\n        \"samples\": [\n          0.877811895562537\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"pergunta_embedding_custom\",\n      \"properties\": {\n        \"dtype\": \"object\",\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"chunk_embedding_custom\",\n      \"properties\": {\n        \"dtype\": \"object\",\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"cosine_similarity_custom\",\n      \"properties\": {\n        \"dtype\": \"float32\",\n        \"num_unique_values\": 140,\n        \"samples\": [\n          0.31233397126197815\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"id\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 40,\n        \"min\": 0,\n        \"max\": 139,\n        \"num_unique_values\": 140,\n        \"samples\": [\n          108\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 42
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_ada002_custom[\"pergunta_embedding_custom\"].iloc[0].shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-fAVJ9zAZH7W",
        "outputId": "f65b6ea7-33a4-42d4-cab6-3f578dd77ca4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(2048,)"
            ]
          },
          "metadata": {},
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def ada002_custom_search(df_ada002_custom):\n",
        "\n",
        "    query_matrix_custom = np.vstack(df_ada002_custom[\"pergunta_embedding_custom\"])\n",
        "    contents_matrix_custom = np.vstack(df_ada002_custom[\"chunk_embedding_custom\"])\n",
        "    similarity_matrix_custom = np.dot(query_matrix_custom, contents_matrix_custom.T)\n",
        "\n",
        "    ids = df_ada002_custom[\"id\"].values.tolist()\n",
        "    similarity_custom = {}\n",
        "    for query_id, hits in zip(ids, similarity_matrix_custom):\n",
        "        if similarity_custom.get(query_id) is None:\n",
        "            similarity_custom[query_id] = []\n",
        "        for doc_id, hit in zip(ids, hits):\n",
        "            similarity_custom[query_id].append((doc_id, hit))\n",
        "\n",
        "    sorted_similarity_custom = {key: sorted(\n",
        "        value, key=lambda x: x[1], reverse=True) for key, value in similarity_custom.items()}\n",
        "\n",
        "    return sorted_similarity_custom"
      ],
      "metadata": {
        "id": "90acsgfp5Ipg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sorted_similarity_custom = ada002_custom_search(df_ada002_custom)\n",
        "\n",
        "# Save run file\n",
        "experiment = \"custom_ada002\"\n",
        "run_file = f\"runs/run.hp.{experiment}.txt\"\n",
        "eval_file = f\"evals/eval.hp.{experiment}.txt\"\n",
        "\n",
        "!rm -rf $run_file\n",
        "!rm -rf $eval_file\n",
        "\n",
        "with open(run_file,'w') as f_out:\n",
        "    for query_id, hits in tqdm(sorted_similarity_custom.items(), desc=f'Writing file on {run_file}'):\n",
        "        rank = 1\n",
        "        for doc_id, score in hits:\n",
        "            f_out.write(f'{query_id}\\tQ0\\t{doc_id}\\t{rank}\\t{score}\\t{\"CUSTOM-ADA002\"}\\n')\n",
        "            rank+=1\n",
        "\n",
        "!/content/pyserini/tools/eval/trec_eval.9.0.4/trec_eval -c -M 10 -m recip_rank -m recall.3,5,7,9 qrels.tsv $run_file > $eval_file 2>&1\n",
        "trec_aux = pd.read_csv(eval_file, sep = \"\\t\", names = [\"metric\", \"all\", \"value\"])\n",
        "trec_aux[\"experiment\"] = experiment\n",
        "trec_results[experiment] = trec_aux.copy()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IxP71E045IsJ",
        "outputId": "ddeede08-dd50-4c8a-cec7-576a8a0a8658"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Writing file on runs/run.hp.custom_ada002.txt: 100%|██████████| 140/140 [00:00<00:00, 4083.18it/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "trec_results[experiment]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "ELxj9KHqXwNa",
        "outputId": "5744f202-740f-400a-c878-2047c6234e21"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                   metric  all   value     experiment\n",
              "0  recip_rank              all  0.6518  custom_ada002\n",
              "1  recall_3                all  0.7500  custom_ada002\n",
              "2  recall_5                all  0.8357  custom_ada002\n",
              "3  recall_7                all  0.9000  custom_ada002\n",
              "4  recall_9                all  0.9143  custom_ada002"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-bb4c3e80-96e6-41a5-b1af-a6967c5f08aa\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>metric</th>\n",
              "      <th>all</th>\n",
              "      <th>value</th>\n",
              "      <th>experiment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>recip_rank</td>\n",
              "      <td>all</td>\n",
              "      <td>0.6518</td>\n",
              "      <td>custom_ada002</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>recall_3</td>\n",
              "      <td>all</td>\n",
              "      <td>0.7500</td>\n",
              "      <td>custom_ada002</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>recall_5</td>\n",
              "      <td>all</td>\n",
              "      <td>0.8357</td>\n",
              "      <td>custom_ada002</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>recall_7</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9000</td>\n",
              "      <td>custom_ada002</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>recall_9</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9143</td>\n",
              "      <td>custom_ada002</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-bb4c3e80-96e6-41a5-b1af-a6967c5f08aa')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-bb4c3e80-96e6-41a5-b1af-a6967c5f08aa button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-bb4c3e80-96e6-41a5-b1af-a6967c5f08aa');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-aade5329-7b40-48a5-8563-342ebd88ce57\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-aade5329-7b40-48a5-8563-342ebd88ce57')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-aade5329-7b40-48a5-8563-342ebd88ce57 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"trec_results[experiment]\",\n  \"rows\": 5,\n  \"fields\": [\n    {\n      \"column\": \"metric\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 5,\n        \"samples\": [\n          \"recall_3              \",\n          \"recall_9              \",\n          \"recall_5              \"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"all\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          \"all\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"value\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.10984713469180704,\n        \"min\": 0.6518,\n        \"max\": 0.9143,\n        \"num_unique_values\": 5,\n        \"samples\": [\n          0.75\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"experiment\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          \"custom_ada002\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 46
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Hybrid Search (BM25 + ADA002-CUSTOM)"
      ],
      "metadata": {
        "id": "gkBpTGdiCMKT"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# save run file\n",
        "rrf_k_list = [60]\n",
        "for rrf_k in rrf_k_list:\n",
        "    experiment = f\"hybrid_bm25_custom_ada002_k_{rrf_k}\"\n",
        "    run_file = f'runs/run.hp.{experiment}.txt'\n",
        "    eval_file = f'evals/eval.hp.{experiment}.txt'\n",
        "\n",
        "    run_file1_path = 'runs/run.hp.bm25.lucene.txt'\n",
        "    run_file2_path = 'runs/run.hp.custom_ada002.txt'\n",
        "\n",
        "    !python -m pyserini.fusion \\\n",
        "      --runs $run_file1_path $run_file2_path \\\n",
        "      --output $run_file \\\n",
        "      --rrf.k $rrf_k \\\n",
        "      --k 1000\n",
        "\n",
        "    !/content/pyserini/tools/eval/trec_eval.9.0.4/trec_eval -c -M 10 -m recip_rank -m recall.3,5,7,9 qrels.tsv $run_file > $eval_file 2>&1\n",
        "    trec_aux = pd.read_csv(eval_file, sep = \"\\t\", names = [\"metric\", \"all\", \"value\"])\n",
        "    trec_aux[\"experiment\"] = experiment\n",
        "    trec_results[experiment] = trec_aux.copy()"
      ],
      "metadata": {
        "id": "NoMGm11nKkWm"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "rrf_k_list = [60]\n",
        "for rrf_k in rrf_k_list:\n",
        "    experiment = f\"hybrid_bm25_custom_ada002_k_{rrf_k}\"\n",
        "    display(trec_results[experiment])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "L1iNbjH6YPc1",
        "outputId": "2d8185d4-cea2-4652-fb3b-4b5fc1a2981e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "                   metric  all   value                      experiment\n",
              "0  recip_rank              all  0.8436  hybrid_bm25_custom_ada002_k_60\n",
              "1  recall_3                all  0.9143  hybrid_bm25_custom_ada002_k_60\n",
              "2  recall_5                all  0.9429  hybrid_bm25_custom_ada002_k_60\n",
              "3  recall_7                all  0.9714  hybrid_bm25_custom_ada002_k_60\n",
              "4  recall_9                all  0.9857  hybrid_bm25_custom_ada002_k_60"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-6641adc0-2590-44c6-a6ef-83a0d5d8c64c\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>metric</th>\n",
              "      <th>all</th>\n",
              "      <th>value</th>\n",
              "      <th>experiment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>recip_rank</td>\n",
              "      <td>all</td>\n",
              "      <td>0.8436</td>\n",
              "      <td>hybrid_bm25_custom_ada002_k_60</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>recall_3</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9143</td>\n",
              "      <td>hybrid_bm25_custom_ada002_k_60</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>recall_5</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9429</td>\n",
              "      <td>hybrid_bm25_custom_ada002_k_60</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>recall_7</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9714</td>\n",
              "      <td>hybrid_bm25_custom_ada002_k_60</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>recall_9</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9857</td>\n",
              "      <td>hybrid_bm25_custom_ada002_k_60</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-6641adc0-2590-44c6-a6ef-83a0d5d8c64c')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-6641adc0-2590-44c6-a6ef-83a0d5d8c64c button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-6641adc0-2590-44c6-a6ef-83a0d5d8c64c');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-39b39db1-06a2-409c-ba63-e6a1f7652e52\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-39b39db1-06a2-409c-ba63-e6a1f7652e52')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-39b39db1-06a2-409c-ba63-e6a1f7652e52 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"    display(trec_results[experiment])\",\n  \"rows\": 5,\n  \"fields\": [\n    {\n      \"column\": \"metric\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 5,\n        \"samples\": [\n          \"recall_3              \",\n          \"recall_9              \",\n          \"recall_5              \"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"all\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          \"all\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"value\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.056306811310888494,\n        \"min\": 0.8436,\n        \"max\": 0.9857,\n        \"num_unique_values\": 5,\n        \"samples\": [\n          0.9143\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"experiment\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          \"hybrid_bm25_custom_ada002_k_60\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Reranker\n",
        "\n",
        "Referências:\n",
        "* https://github.com/zetaalphavector/InPars/blob/master/inpars/rerank.py\n",
        "* https://huggingface.co/unicamp-dl/mt5-base-en-pt-msmarco-v2\n",
        "* https://github.com/castorini/pygaggle/blob/master/pygaggle/rerank/transformer.py\n",
        "* https://github.com/castorini/pygaggle/blob/08339dd31f58ef40fbaa109726402e164eeba125/pygaggle/run/robust04_reranker_pipeline_gpu.py"
      ],
      "metadata": {
        "id": "4Y9o8r_-qkFp"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import csv\n",
        "import argparse\n",
        "import torch\n",
        "from math import ceil, exp\n",
        "from typing import List\n",
        "from transformers import (\n",
        "    AutoConfig,\n",
        "    AutoTokenizer,\n",
        "    AutoModelForSequenceClassification,\n",
        "    AutoModelForSeq2SeqLM,\n",
        "    T5ForConditionalGeneration,\n",
        "    T5Tokenizer,\n",
        "    MT5ForConditionalGeneration\n",
        ")"
      ],
      "metadata": {
        "id": "Afv6C5qQqjgG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "prediction_tokens = {\n",
        "        'unicamp-dl/mt5-base-en-pt-msmarco-v2':      ['▁no'   , '▁yes'],\n",
        "        }"
      ],
      "metadata": {
        "id": "barbDaSjI0y4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def chunks(lst, n):\n",
        "    for i in range(0, len(lst), n):\n",
        "        yield lst[i : i + n]\n",
        "\n",
        "class Reranker:\n",
        "    def __init__(self, silent=False, batch_size=8, fp16=False, torchscript=False, device=None):\n",
        "        self.silent = silent\n",
        "        self.batch_size = batch_size\n",
        "        self.fp16 = fp16\n",
        "        self.torchscript = torchscript\n",
        "        self.device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "    @classmethod\n",
        "    def from_pretrained(cls, model_name_or_path, **kwargs):\n",
        "        config = AutoConfig.from_pretrained(model_name_or_path)\n",
        "        print(f\"architecture: {config.architectures}\")\n",
        "        print(f\"architecture: MonoT5Reranker\")\n",
        "        return MonoT5Reranker(model_name_or_path, **kwargs)\n",
        "\n",
        "class MonoT5Reranker(Reranker):\n",
        "    name: str = 'MonoT5'\n",
        "    prompt_template: str = \"Query: {query} Document: {text} Relevant:\"\n",
        "\n",
        "    def __init__(\n",
        "        self,\n",
        "        model_name_or_path='unicamp-dl/mt5-base-en-pt-msmarco-v2',\n",
        "        token_false=None,\n",
        "        token_true=True,\n",
        "        torch_compile=False,\n",
        "        **kwargs\n",
        "    ):\n",
        "        super().__init__(**kwargs)\n",
        "        if not self.device:\n",
        "            self.device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "            print(self.device)\n",
        "        model_args = {}\n",
        "        if self.fp16:\n",
        "            model_args[\"torch_dtype\"] = torch.float16\n",
        "        self.model = MT5ForConditionalGeneration.from_pretrained(model_name_or_path, **model_args)\n",
        "        self.torch_compile = torch_compile\n",
        "        if torch_compile:\n",
        "            self.model = torch.compile(self.model)\n",
        "        self.model.to(self.device)\n",
        "        self.tokenizer = T5Tokenizer.from_pretrained(model_name_or_path)\n",
        "        self.token_false_id, self.token_true_id = self.get_prediction_tokens(\n",
        "            model_name_or_path, self.tokenizer, token_false, token_true,\n",
        "        )\n",
        "\n",
        "    def get_prediction_tokens(self, model_name_or_path, tokenizer, token_false=None, token_true=None):\n",
        "        if not (token_false and token_true):\n",
        "            if model_name_or_path in prediction_tokens:\n",
        "                token_false, token_true = prediction_tokens[model_name_or_path]\n",
        "                token_false_id = tokenizer.get_vocab()[token_false]\n",
        "                token_true_id  = tokenizer.get_vocab()[token_true]\n",
        "                return token_false_id, token_true_id\n",
        "            else:\n",
        "                # raise Exception(f\"We don't know the indexes for the non-relevant/relevant tokens for\\\n",
        "                #         the checkpoint {model_name_or_path} and you did not provide any.\")\n",
        "                return self.get_prediction_tokens('castorini/monot5-base-msmarco', self.tokenizer)\n",
        "        else:\n",
        "            token_false_id = tokenizer.get_vocab()[token_false]\n",
        "            token_true_id  = tokenizer.get_vocab()[token_true]\n",
        "            return token_false_id, token_true_id\n",
        "\n",
        "    @torch.inference_mode()\n",
        "    def rescore(self, pairs: List[List[str]]):\n",
        "        scores = []\n",
        "        for batch in tqdm(\n",
        "            chunks(pairs, self.batch_size),\n",
        "            disable=self.silent,\n",
        "            desc=\"Rescoring\",\n",
        "            total=ceil(len(pairs) / self.batch_size),\n",
        "        ):\n",
        "            prompts = [\n",
        "                self.prompt_template.format(query=query, text=text)\n",
        "                for (query, text) in batch\n",
        "            ]\n",
        "            tokens = self.tokenizer(\n",
        "                prompts,\n",
        "                padding=True,\n",
        "                truncation=True,\n",
        "                return_tensors=\"pt\",\n",
        "                max_length=self.tokenizer.model_max_length,\n",
        "                pad_to_multiple_of=(8 if self.torch_compile else None),\n",
        "            ).to(self.device)\n",
        "            output = self.model.generate(\n",
        "                **tokens,\n",
        "                max_new_tokens=1,\n",
        "                return_dict_in_generate=True,\n",
        "                output_scores=True,\n",
        "            )\n",
        "            batch_scores = output.scores[0]\n",
        "            batch_scores = batch_scores[:, [self.token_false_id, self.token_true_id]]\n",
        "            batch_scores = torch.nn.functional.log_softmax(batch_scores, dim=1)\n",
        "            scores += batch_scores[:, 1].exp().tolist()\n",
        "        return scores\n",
        "\n",
        "class TRECRun:\n",
        "    def __init__(self, run_file, sep=r\"\\s+\"):\n",
        "\n",
        "        self.run_file = run_file\n",
        "        self.df = pd.read_csv(\n",
        "            run_file,\n",
        "            sep=sep,\n",
        "            quoting=csv.QUOTE_NONE,\n",
        "            keep_default_na=False,\n",
        "            names=(\"qid\", \"_1\", \"docid\", \"rank\", \"score\", \"ranker\"),\n",
        "            dtype=str,\n",
        "        )\n",
        "\n",
        "    def rerank(self, ranker, queries, corpus, top_k=1000):\n",
        "        # Converts run to float32 and subtracts a large number to ensure the BM25 scores\n",
        "        # are lower than those provided by the neural ranker.\n",
        "        self.df[\"score\"] = (\n",
        "            self.df[\"score\"]\n",
        "            .astype(\"float32\")\n",
        "            .apply(lambda x: x-10000)\n",
        "        )\n",
        "\n",
        "        # Reranks only the top-k documents for each query\n",
        "        subset = (\n",
        "            self.df[[\"qid\", \"docid\"]]\n",
        "            .groupby(\"qid\")\n",
        "            .head(top_k)\n",
        "            .apply(lambda x: [queries[int(x[\"qid\"])], corpus[int(x[\"docid\"])]], axis=1)\n",
        "        )\n",
        "        scores = ranker.rescore(subset.values.tolist())\n",
        "\n",
        "        self.df.loc[subset.index, \"score\"] = scores\n",
        "\n",
        "        self.df[\"ranker\"] = ranker.name\n",
        "        self.df = (\n",
        "            self.df\n",
        "            .groupby(\"qid\")\n",
        "            .apply(lambda x: x.sort_values(\"score\", ascending=False))\n",
        "            .reset_index(drop=True)\n",
        "        )\n",
        "\n",
        "        self.df[\"rank\"] = self.df.groupby(\"qid\").cumcount() + 1\n",
        "\n",
        "    def save(self, path):\n",
        "        self.df.to_csv(path, index=False, sep=\"\\t\", header=False, float_format='%.15f')"
      ],
      "metadata": {
        "id": "8MPLfXd2KY_V"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def parse_args(args):\n",
        "    parser = argparse.ArgumentParser()\n",
        "\n",
        "    parser.add_argument(\n",
        "        \"--model\", default='castorini/monot5-small-msmarco-100k', type=str, required=False, help=\"Reranker model.\"\n",
        "        )\n",
        "    parser.add_argument(\n",
        "        \"--input_run\", default=None, type=str, help=\"Initial run to be reranked.\"\n",
        "        )\n",
        "    parser.add_argument(\n",
        "        \"--output_run\", default=None, type=str, required=True, help=\"Path to save the reranked run.\"\n",
        "        )\n",
        "    parser.add_argument(\n",
        "        \"--fp16\", default=True, type=bool, help=\"Whether to use FP16 weights during inference.\"\n",
        "        )\n",
        "    parser.add_argument(\n",
        "        \"--torch_compile\", default=True, type=bool, help=\"Whether to compile the model with `torch.compile`.\"\n",
        "        )\n",
        "    parser.add_argument(\n",
        "        \"--batch_size\", default=16, type=int, help=\"Batch size for inference.\"\n",
        "        )\n",
        "    parser.add_argument(\n",
        "        \"--top_k\", default=1_000, type=int, help=\"Top-k documents to be reranked for each query.\"\n",
        "        )\n",
        "    return parser.parse_args(args)"
      ],
      "metadata": {
        "id": "6AAmrbc5NBAe"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Save run file\n",
        "experiment = \"mt5-base-en-pt-msmarco-v2-reranker\"\n",
        "run_file = f\"runs/run.hp.{experiment}.txt\"\n",
        "eval_file = f\"evals/eval.hp.{experiment}.txt\"\n",
        "\n",
        "def dev_parser():\n",
        "    parser = parse_args([\n",
        "        '--model', 'unicamp-dl/mt5-base-en-pt-msmarco-v2',\n",
        "        '--input_run', 'runs/run.hp.bm25.lucene.txt',\n",
        "        '--output_run', run_file,\n",
        "        '--batch_size', f'16',\n",
        "        '--top_k', f'50'\n",
        "        ])\n",
        "    return parser"
      ],
      "metadata": {
        "id": "QZO4qXhY8Xs6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "args = dev_parser()\n",
        "print(args)\n",
        "input_run = args.input_run\n",
        "top_k = args.top_k\n",
        "output_run = args.output_run\n",
        "\n",
        "model = Reranker.from_pretrained(\n",
        "    model_name_or_path=args.model,\n",
        "    batch_size=args.batch_size,\n",
        "    fp16=args.fp16,\n",
        "    torch_compile=args.torch_compile\n",
        ")\n",
        "\n",
        "!rm -rf $run_file\n",
        "!rm -rf $eval_file\n",
        "\n",
        "run = TRECRun(input_run)\n",
        "run.rerank(model, topics, corpus, top_k=top_k)\n",
        "run.save(output_run)\n",
        "\n",
        "!/content/pyserini/tools/eval/trec_eval.9.0.4/trec_eval -c -M 10 -m recip_rank -m recall.3,5,7,9 qrels.tsv $run_file > $eval_file 2>&1\n",
        "trec_aux = pd.read_csv(eval_file, sep = \"\\t\", names = [\"metric\", \"all\", \"value\"])\n",
        "trec_aux[\"experiment\"] = experiment\n",
        "trec_results[experiment] = trec_aux.copy()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aMtdhcSW4wuP",
        "outputId": "7a9aba35-3aa3-44d8-b236-427f8ffc1060"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Namespace(model='unicamp-dl/mt5-base-en-pt-msmarco-v2', input_run='runs/run.hp.bm25.lucene.txt', output_run='runs/run.hp.mt5-base-en-pt-msmarco-v2-reranker.txt', fp16=True, torch_compile=True, batch_size=16, top_k=50)\n",
            "architecture: ['MT5ForConditionalGeneration']\n",
            "architecture: MonoT5Reranker\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/lib/python3.10/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock.\n",
            "  self.pid = os.fork()\n",
            "You are using the default legacy behaviour of the <class 'transformers.models.t5.tokenization_t5.T5Tokenizer'>. This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565\n",
            "Rescoring: 100%|██████████| 438/438 [03:26<00:00,  2.12it/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "trec_results[experiment]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "w1asixLM1pQ0",
        "outputId": "e59ce13c-ec55-4498-c8c2-c38721de864a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                   metric  all   value                          experiment\n",
              "0  recip_rank              all  0.9196  mt5-base-en-pt-msmarco-v2-reranker\n",
              "1  recall_3                all  0.9714  mt5-base-en-pt-msmarco-v2-reranker\n",
              "2  recall_5                all  0.9857  mt5-base-en-pt-msmarco-v2-reranker\n",
              "3  recall_7                all  0.9929  mt5-base-en-pt-msmarco-v2-reranker\n",
              "4  recall_9                all  1.0000  mt5-base-en-pt-msmarco-v2-reranker"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-a871f02e-0cfa-4905-99b5-94044b21313c\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>metric</th>\n",
              "      <th>all</th>\n",
              "      <th>value</th>\n",
              "      <th>experiment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>recip_rank</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9196</td>\n",
              "      <td>mt5-base-en-pt-msmarco-v2-reranker</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>recall_3</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9714</td>\n",
              "      <td>mt5-base-en-pt-msmarco-v2-reranker</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>recall_5</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9857</td>\n",
              "      <td>mt5-base-en-pt-msmarco-v2-reranker</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>recall_7</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9929</td>\n",
              "      <td>mt5-base-en-pt-msmarco-v2-reranker</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>recall_9</td>\n",
              "      <td>all</td>\n",
              "      <td>1.0000</td>\n",
              "      <td>mt5-base-en-pt-msmarco-v2-reranker</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-a871f02e-0cfa-4905-99b5-94044b21313c')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-a871f02e-0cfa-4905-99b5-94044b21313c button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-a871f02e-0cfa-4905-99b5-94044b21313c');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-1379ba2c-1f59-4a7c-8d84-b947c0ab8e15\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-1379ba2c-1f59-4a7c-8d84-b947c0ab8e15')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-1379ba2c-1f59-4a7c-8d84-b947c0ab8e15 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"trec_results[experiment]\",\n  \"rows\": 5,\n  \"fields\": [\n    {\n      \"column\": \"metric\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 5,\n        \"samples\": [\n          \"recall_3              \",\n          \"recall_9              \",\n          \"recall_5              \"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"all\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          \"all\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"value\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.03215660118855848,\n        \"min\": 0.9196,\n        \"max\": 1.0,\n        \"num_unique_values\": 5,\n        \"samples\": [\n          0.9714\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"experiment\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          \"mt5-base-en-pt-msmarco-v2-reranker\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 55
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "y9yEtqdTo-JG"
      },
      "source": [
        "## Compile results"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "results = pd.DataFrame()\n",
        "for experiment in trec_results.keys():\n",
        "    results = pd.concat([results, trec_results[experiment]], axis = 0)\n",
        "\n",
        "results = results[~results[\"experiment\"].isin([\"0.0_bm25_1.0_ada002\", \"1.0_bm25_0.0_ada002\", \"0.0_bm25_1.0_ada002_custom\", \"1.0_bm25_0.0_ada002_custom\"])].copy()"
      ],
      "metadata": {
        "id": "FMsO9dcQQ3Xc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "BbkXDK92oTT9",
        "outputId": "10252002-4ad0-432a-dae1-fd3af9561b8f"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "                   metric  all   value                          experiment\n",
              "0  recip_rank              all  0.9196  mt5-base-en-pt-msmarco-v2-reranker\n",
              "0  recip_rank              all  0.8794                         bm25.lucene\n",
              "0  recip_rank              all  0.8436      hybrid_bm25_custom_ada002_k_60\n",
              "0  recip_rank              all  0.7587             hybrid_bm25_ada002_k_60\n",
              "0  recip_rank              all  0.6518                       custom_ada002\n",
              "0  recip_rank              all  0.5645                              ada002"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-475ee886-e1b7-4a14-8c3f-4ead7dd235f8\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>metric</th>\n",
              "      <th>all</th>\n",
              "      <th>value</th>\n",
              "      <th>experiment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>recip_rank</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9196</td>\n",
              "      <td>mt5-base-en-pt-msmarco-v2-reranker</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>recip_rank</td>\n",
              "      <td>all</td>\n",
              "      <td>0.8794</td>\n",
              "      <td>bm25.lucene</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>recip_rank</td>\n",
              "      <td>all</td>\n",
              "      <td>0.8436</td>\n",
              "      <td>hybrid_bm25_custom_ada002_k_60</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>recip_rank</td>\n",
              "      <td>all</td>\n",
              "      <td>0.7587</td>\n",
              "      <td>hybrid_bm25_ada002_k_60</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>recip_rank</td>\n",
              "      <td>all</td>\n",
              "      <td>0.6518</td>\n",
              "      <td>custom_ada002</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>recip_rank</td>\n",
              "      <td>all</td>\n",
              "      <td>0.5645</td>\n",
              "      <td>ada002</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-475ee886-e1b7-4a14-8c3f-4ead7dd235f8')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-475ee886-e1b7-4a14-8c3f-4ead7dd235f8 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-475ee886-e1b7-4a14-8c3f-4ead7dd235f8');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-0740bfa4-b1b6-4457-b718-7393ca98aaad\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-0740bfa4-b1b6-4457-b718-7393ca98aaad')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-0740bfa4-b1b6-4457-b718-7393ca98aaad button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"    display(results[results[\\\"metric\\\"]\",\n  \"rows\": 6,\n  \"fields\": [\n    {\n      \"column\": \"metric\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          \"recip_rank            \"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"all\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          \"all\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"value\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.13863167026332762,\n        \"min\": 0.5645,\n        \"max\": 0.9196,\n        \"num_unique_values\": 6,\n        \"samples\": [\n          0.9196\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"experiment\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 6,\n        \"samples\": [\n          \"mt5-base-en-pt-msmarco-v2-reranker\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "                   metric  all   value                          experiment\n",
              "1  recall_3                all  0.9714  mt5-base-en-pt-msmarco-v2-reranker\n",
              "1  recall_3                all  0.9143                         bm25.lucene\n",
              "1  recall_3                all  0.9143      hybrid_bm25_custom_ada002_k_60\n",
              "1  recall_3                all  0.8357             hybrid_bm25_ada002_k_60\n",
              "1  recall_3                all  0.7500                       custom_ada002\n",
              "1  recall_3                all  0.6214                              ada002"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-d792f590-c611-4b8d-9a58-7c774c849c69\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>metric</th>\n",
              "      <th>all</th>\n",
              "      <th>value</th>\n",
              "      <th>experiment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>recall_3</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9714</td>\n",
              "      <td>mt5-base-en-pt-msmarco-v2-reranker</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>recall_3</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9143</td>\n",
              "      <td>bm25.lucene</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>recall_3</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9143</td>\n",
              "      <td>hybrid_bm25_custom_ada002_k_60</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>recall_3</td>\n",
              "      <td>all</td>\n",
              "      <td>0.8357</td>\n",
              "      <td>hybrid_bm25_ada002_k_60</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>recall_3</td>\n",
              "      <td>all</td>\n",
              "      <td>0.7500</td>\n",
              "      <td>custom_ada002</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>recall_3</td>\n",
              "      <td>all</td>\n",
              "      <td>0.6214</td>\n",
              "      <td>ada002</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-d792f590-c611-4b8d-9a58-7c774c849c69')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-d792f590-c611-4b8d-9a58-7c774c849c69 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-d792f590-c611-4b8d-9a58-7c774c849c69');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-c91ba893-927b-4e2b-9924-9bac14a0e593\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-c91ba893-927b-4e2b-9924-9bac14a0e593')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-c91ba893-927b-4e2b-9924-9bac14a0e593 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"    display(results[results[\\\"metric\\\"]\",\n  \"rows\": 6,\n  \"fields\": [\n    {\n      \"column\": \"metric\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          \"recall_3              \"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"all\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          \"all\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"value\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.12963871978181007,\n        \"min\": 0.6214,\n        \"max\": 0.9714,\n        \"num_unique_values\": 5,\n        \"samples\": [\n          0.9143\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"experiment\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 6,\n        \"samples\": [\n          \"mt5-base-en-pt-msmarco-v2-reranker\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "                   metric  all   value                          experiment\n",
              "2  recall_5                all  0.9857  mt5-base-en-pt-msmarco-v2-reranker\n",
              "2  recall_5                all  0.9714                         bm25.lucene\n",
              "2  recall_5                all  0.9429      hybrid_bm25_custom_ada002_k_60\n",
              "2  recall_5                all  0.8786             hybrid_bm25_ada002_k_60\n",
              "2  recall_5                all  0.8357                       custom_ada002\n",
              "2  recall_5                all  0.6929                              ada002"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-e046f008-1cb9-40bb-9baf-9c91d330c119\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>metric</th>\n",
              "      <th>all</th>\n",
              "      <th>value</th>\n",
              "      <th>experiment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>recall_5</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9857</td>\n",
              "      <td>mt5-base-en-pt-msmarco-v2-reranker</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>recall_5</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9714</td>\n",
              "      <td>bm25.lucene</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>recall_5</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9429</td>\n",
              "      <td>hybrid_bm25_custom_ada002_k_60</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>recall_5</td>\n",
              "      <td>all</td>\n",
              "      <td>0.8786</td>\n",
              "      <td>hybrid_bm25_ada002_k_60</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>recall_5</td>\n",
              "      <td>all</td>\n",
              "      <td>0.8357</td>\n",
              "      <td>custom_ada002</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>recall_5</td>\n",
              "      <td>all</td>\n",
              "      <td>0.6929</td>\n",
              "      <td>ada002</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e046f008-1cb9-40bb-9baf-9c91d330c119')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-e046f008-1cb9-40bb-9baf-9c91d330c119 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-e046f008-1cb9-40bb-9baf-9c91d330c119');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-ce8f02fe-3d63-4cac-88c2-6d468f8601af\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-ce8f02fe-3d63-4cac-88c2-6d468f8601af')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-ce8f02fe-3d63-4cac-88c2-6d468f8601af button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"    display(results[results[\\\"metric\\\"]\",\n  \"rows\": 6,\n  \"fields\": [\n    {\n      \"column\": \"metric\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          \"recall_5              \"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"all\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          \"all\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"value\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.10984572211363841,\n        \"min\": 0.6929,\n        \"max\": 0.9857,\n        \"num_unique_values\": 6,\n        \"samples\": [\n          0.9857\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"experiment\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 6,\n        \"samples\": [\n          \"mt5-base-en-pt-msmarco-v2-reranker\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "                   metric  all   value                          experiment\n",
              "3  recall_7                all  0.9929  mt5-base-en-pt-msmarco-v2-reranker\n",
              "3  recall_7                all  0.9857                         bm25.lucene\n",
              "3  recall_7                all  0.9714      hybrid_bm25_custom_ada002_k_60\n",
              "3  recall_7                all  0.9143             hybrid_bm25_ada002_k_60\n",
              "3  recall_7                all  0.9000                       custom_ada002\n",
              "3  recall_7                all  0.7500                              ada002"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-0022a2eb-c67e-4b8b-ba21-3288d828f4e5\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>metric</th>\n",
              "      <th>all</th>\n",
              "      <th>value</th>\n",
              "      <th>experiment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>recall_7</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9929</td>\n",
              "      <td>mt5-base-en-pt-msmarco-v2-reranker</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>recall_7</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9857</td>\n",
              "      <td>bm25.lucene</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>recall_7</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9714</td>\n",
              "      <td>hybrid_bm25_custom_ada002_k_60</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>recall_7</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9143</td>\n",
              "      <td>hybrid_bm25_ada002_k_60</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>recall_7</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9000</td>\n",
              "      <td>custom_ada002</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>recall_7</td>\n",
              "      <td>all</td>\n",
              "      <td>0.7500</td>\n",
              "      <td>ada002</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-0022a2eb-c67e-4b8b-ba21-3288d828f4e5')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-0022a2eb-c67e-4b8b-ba21-3288d828f4e5 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-0022a2eb-c67e-4b8b-ba21-3288d828f4e5');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-2cce65d6-a82f-4e62-bb7b-dc3c11ac6f22\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-2cce65d6-a82f-4e62-bb7b-dc3c11ac6f22')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-2cce65d6-a82f-4e62-bb7b-dc3c11ac6f22 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"    display(results[results[\\\"metric\\\"]\",\n  \"rows\": 6,\n  \"fields\": [\n    {\n      \"column\": \"metric\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          \"recall_7              \"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"all\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          \"all\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"value\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.09121396274693915,\n        \"min\": 0.75,\n        \"max\": 0.9929,\n        \"num_unique_values\": 6,\n        \"samples\": [\n          0.9929\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"experiment\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 6,\n        \"samples\": [\n          \"mt5-base-en-pt-msmarco-v2-reranker\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "                   metric  all   value                          experiment\n",
              "4  recall_9                all  1.0000  mt5-base-en-pt-msmarco-v2-reranker\n",
              "4  recall_9                all  0.9857                         bm25.lucene\n",
              "4  recall_9                all  0.9857      hybrid_bm25_custom_ada002_k_60\n",
              "4  recall_9                all  0.9571             hybrid_bm25_ada002_k_60\n",
              "4  recall_9                all  0.9143                       custom_ada002\n",
              "4  recall_9                all  0.8071                              ada002"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-be9a4445-ccd8-48f2-986c-d7efb921ba67\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>metric</th>\n",
              "      <th>all</th>\n",
              "      <th>value</th>\n",
              "      <th>experiment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>recall_9</td>\n",
              "      <td>all</td>\n",
              "      <td>1.0000</td>\n",
              "      <td>mt5-base-en-pt-msmarco-v2-reranker</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>recall_9</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9857</td>\n",
              "      <td>bm25.lucene</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>recall_9</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9857</td>\n",
              "      <td>hybrid_bm25_custom_ada002_k_60</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>recall_9</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9571</td>\n",
              "      <td>hybrid_bm25_ada002_k_60</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>recall_9</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9143</td>\n",
              "      <td>custom_ada002</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>recall_9</td>\n",
              "      <td>all</td>\n",
              "      <td>0.8071</td>\n",
              "      <td>ada002</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-be9a4445-ccd8-48f2-986c-d7efb921ba67')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-be9a4445-ccd8-48f2-986c-d7efb921ba67 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-be9a4445-ccd8-48f2-986c-d7efb921ba67');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-94a4ec5e-f14f-4aad-81e9-c189906db771\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-94a4ec5e-f14f-4aad-81e9-c189906db771')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-94a4ec5e-f14f-4aad-81e9-c189906db771 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"    display(results[results[\\\"metric\\\"]\",\n  \"rows\": 6,\n  \"fields\": [\n    {\n      \"column\": \"metric\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          \"recall_9              \"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"all\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          \"all\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"value\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.07263051011799379,\n        \"min\": 0.8071,\n        \"max\": 1.0,\n        \"num_unique_values\": 5,\n        \"samples\": [\n          0.9857\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"experiment\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 6,\n        \"samples\": [\n          \"mt5-base-en-pt-msmarco-v2-reranker\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n"
          ]
        }
      ],
      "source": [
        "metric  = 'all' #@param ['all', 'recip_rank', 'recall_3', 'recall_5', 'recall_7','recall_9'] {allow-input: true}\n",
        "\n",
        "if metric == \"all\":\n",
        "    for metric in results[\"metric\"].unique():\n",
        "        display(results[results[\"metric\"].str.strip() == metric.strip()].sort_values(by = \"value\", ascending = False))\n",
        "        print(\"\\n\")\n",
        "else:\n",
        "    display(results[results[\"metric\"].str.strip() == metric].sort_values(by = \"value\", ascending = False))"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## context to LLM"
      ],
      "metadata": {
        "id": "W6zkNiGsTxvt"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "file_run_reranker    = pd.read_csv(\"runs/run.hp.mt5-base-en-pt-msmarco-v2-reranker.txt\", sep = \"\\t\", header = None, names = [\"query_id\", \"q0\", \"doc_id\", \"rank\", \"score\", \"system\"])\n",
        "file_run_hybrid      = pd.read_csv(\"runs/run.hp.hybrid_bm25_custom_ada002_k_60.txt\", sep = \"\\t\", header = None, names = [\"query_id\", \"q0\", \"doc_id\", \"rank\", \"score\", \"system\"])\n",
        "file_run_bm25        = pd.read_csv(\"runs/run.hp.bm25.lucene.txt\", sep = \" \", header = None, names = [\"query_id\", \"q0\", \"doc_id\", \"rank\", \"score\", \"system\"])\n",
        "file_run_ada_custom  = pd.read_csv(\"runs/run.hp.custom_ada002.txt\", sep = \"\\t\", header = None, names = [\"query_id\", \"q0\", \"doc_id\", \"rank\", \"score\", \"system\"])\n",
        "file_run_ada         = pd.read_csv(\"runs/run.hp.ada002.txt\", sep = \"\\t\", header = None, names = [\"query_id\", \"q0\", \"doc_id\", \"rank\", \"score\", \"system\"])"
      ],
      "metadata": {
        "id": "lvaEHn3yI8KS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "file_run_ada"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 423
        },
        "id": "hW33o47sER42",
        "outputId": "e86c9345-4291-4016-d402-e5fca292613c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "       query_id  q0  doc_id  rank     score  system\n",
              "0             0  Q0       0     1  0.873560  ADA002\n",
              "1             0  Q0       4     2  0.861825  ADA002\n",
              "2             0  Q0      23     3  0.861104  ADA002\n",
              "3             0  Q0     103     4  0.856059  ADA002\n",
              "4             0  Q0      25     5  0.855472  ADA002\n",
              "...         ...  ..     ...   ...       ...     ...\n",
              "19595       139  Q0      36   136  0.816049  ADA002\n",
              "19596       139  Q0     132   137  0.814769  ADA002\n",
              "19597       139  Q0      83   138  0.814735  ADA002\n",
              "19598       139  Q0      30   139  0.814437  ADA002\n",
              "19599       139  Q0      52   140  0.811439  ADA002\n",
              "\n",
              "[19600 rows x 6 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-97ccaaf3-26e0-4289-8d82-d5efcb4ea90c\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>query_id</th>\n",
              "      <th>q0</th>\n",
              "      <th>doc_id</th>\n",
              "      <th>rank</th>\n",
              "      <th>score</th>\n",
              "      <th>system</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>Q0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0.873560</td>\n",
              "      <td>ADA002</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "      <td>Q0</td>\n",
              "      <td>4</td>\n",
              "      <td>2</td>\n",
              "      <td>0.861825</td>\n",
              "      <td>ADA002</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0</td>\n",
              "      <td>Q0</td>\n",
              "      <td>23</td>\n",
              "      <td>3</td>\n",
              "      <td>0.861104</td>\n",
              "      <td>ADA002</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0</td>\n",
              "      <td>Q0</td>\n",
              "      <td>103</td>\n",
              "      <td>4</td>\n",
              "      <td>0.856059</td>\n",
              "      <td>ADA002</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0</td>\n",
              "      <td>Q0</td>\n",
              "      <td>25</td>\n",
              "      <td>5</td>\n",
              "      <td>0.855472</td>\n",
              "      <td>ADA002</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19595</th>\n",
              "      <td>139</td>\n",
              "      <td>Q0</td>\n",
              "      <td>36</td>\n",
              "      <td>136</td>\n",
              "      <td>0.816049</td>\n",
              "      <td>ADA002</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19596</th>\n",
              "      <td>139</td>\n",
              "      <td>Q0</td>\n",
              "      <td>132</td>\n",
              "      <td>137</td>\n",
              "      <td>0.814769</td>\n",
              "      <td>ADA002</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19597</th>\n",
              "      <td>139</td>\n",
              "      <td>Q0</td>\n",
              "      <td>83</td>\n",
              "      <td>138</td>\n",
              "      <td>0.814735</td>\n",
              "      <td>ADA002</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19598</th>\n",
              "      <td>139</td>\n",
              "      <td>Q0</td>\n",
              "      <td>30</td>\n",
              "      <td>139</td>\n",
              "      <td>0.814437</td>\n",
              "      <td>ADA002</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19599</th>\n",
              "      <td>139</td>\n",
              "      <td>Q0</td>\n",
              "      <td>52</td>\n",
              "      <td>140</td>\n",
              "      <td>0.811439</td>\n",
              "      <td>ADA002</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>19600 rows × 6 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-97ccaaf3-26e0-4289-8d82-d5efcb4ea90c')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-97ccaaf3-26e0-4289-8d82-d5efcb4ea90c button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-97ccaaf3-26e0-4289-8d82-d5efcb4ea90c');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-3929df4a-3aff-4826-bc02-e47bd324feaa\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-3929df4a-3aff-4826-bc02-e47bd324feaa')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-3929df4a-3aff-4826-bc02-e47bd324feaa button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "  <div id=\"id_8a95eb94-6efa-4b1e-b23c-d0cb6ab7d1b6\">\n",
              "    <style>\n",
              "      .colab-df-generate {\n",
              "        background-color: #E8F0FE;\n",
              "        border: none;\n",
              "        border-radius: 50%;\n",
              "        cursor: pointer;\n",
              "        display: none;\n",
              "        fill: #1967D2;\n",
              "        height: 32px;\n",
              "        padding: 0 0 0 0;\n",
              "        width: 32px;\n",
              "      }\n",
              "\n",
              "      .colab-df-generate:hover {\n",
              "        background-color: #E2EBFA;\n",
              "        box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "        fill: #174EA6;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate {\n",
              "        background-color: #3B4455;\n",
              "        fill: #D2E3FC;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate:hover {\n",
              "        background-color: #434B5C;\n",
              "        box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "        filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "        fill: #FFFFFF;\n",
              "      }\n",
              "    </style>\n",
              "    <button class=\"colab-df-generate\" onclick=\"generateWithVariable('file_run_ada')\"\n",
              "            title=\"Generate code using this dataframe.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M7,19H8.4L18.45,9,17,7.55,7,17.6ZM5,21V16.75L18.45,3.32a2,2,0,0,1,2.83,0l1.4,1.43a1.91,1.91,0,0,1,.58,1.4,1.91,1.91,0,0,1-.58,1.4L9.25,21ZM18.45,9,17,7.55Zm-12,3A5.31,5.31,0,0,0,4.9,8.1,5.31,5.31,0,0,0,1,6.5,5.31,5.31,0,0,0,4.9,4.9,5.31,5.31,0,0,0,6.5,1,5.31,5.31,0,0,0,8.1,4.9,5.31,5.31,0,0,0,12,6.5,5.46,5.46,0,0,0,6.5,12Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "    <script>\n",
              "      (() => {\n",
              "      const buttonEl =\n",
              "        document.querySelector('#id_8a95eb94-6efa-4b1e-b23c-d0cb6ab7d1b6 button.colab-df-generate');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      buttonEl.onclick = () => {\n",
              "        google.colab.notebook.generateWithVariable('file_run_ada');\n",
              "      }\n",
              "      })();\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "file_run_ada",
              "summary": "{\n  \"name\": \"file_run_ada\",\n  \"rows\": 19600,\n  \"fields\": [\n    {\n      \"column\": \"query_id\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 40,\n        \"min\": 0,\n        \"max\": 139,\n        \"num_unique_values\": 140,\n        \"samples\": [\n          108,\n          67,\n          31\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"q0\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          \"Q0\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"doc_id\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 40,\n        \"min\": 0,\n        \"max\": 139,\n        \"num_unique_values\": 140,\n        \"samples\": [\n          60\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"rank\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 40,\n        \"min\": 1,\n        \"max\": 140,\n        \"num_unique_values\": 140,\n        \"samples\": [\n          109\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"score\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.02259630222915543,\n        \"min\": 0.7274358745746111,\n        \"max\": 0.9053969111406552,\n        \"num_unique_values\": 19600,\n        \"samples\": [\n          0.8464337797770978\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"system\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          \"ADA002\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 59
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "file_run_ada"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 423
        },
        "id": "H2S0u6KiEjPy",
        "outputId": "83b75b4c-87f6-42d2-dae1-0c6326641880"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "       query_id  q0  doc_id  rank     score  system\n",
              "0             0  Q0       0     1  0.873560  ADA002\n",
              "1             0  Q0       4     2  0.861825  ADA002\n",
              "2             0  Q0      23     3  0.861104  ADA002\n",
              "3             0  Q0     103     4  0.856059  ADA002\n",
              "4             0  Q0      25     5  0.855472  ADA002\n",
              "...         ...  ..     ...   ...       ...     ...\n",
              "19595       139  Q0      36   136  0.816049  ADA002\n",
              "19596       139  Q0     132   137  0.814769  ADA002\n",
              "19597       139  Q0      83   138  0.814735  ADA002\n",
              "19598       139  Q0      30   139  0.814437  ADA002\n",
              "19599       139  Q0      52   140  0.811439  ADA002\n",
              "\n",
              "[19600 rows x 6 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-8e75fbde-07f7-483b-a8a7-531b61ff6e38\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>query_id</th>\n",
              "      <th>q0</th>\n",
              "      <th>doc_id</th>\n",
              "      <th>rank</th>\n",
              "      <th>score</th>\n",
              "      <th>system</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>Q0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0.873560</td>\n",
              "      <td>ADA002</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "      <td>Q0</td>\n",
              "      <td>4</td>\n",
              "      <td>2</td>\n",
              "      <td>0.861825</td>\n",
              "      <td>ADA002</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0</td>\n",
              "      <td>Q0</td>\n",
              "      <td>23</td>\n",
              "      <td>3</td>\n",
              "      <td>0.861104</td>\n",
              "      <td>ADA002</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0</td>\n",
              "      <td>Q0</td>\n",
              "      <td>103</td>\n",
              "      <td>4</td>\n",
              "      <td>0.856059</td>\n",
              "      <td>ADA002</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0</td>\n",
              "      <td>Q0</td>\n",
              "      <td>25</td>\n",
              "      <td>5</td>\n",
              "      <td>0.855472</td>\n",
              "      <td>ADA002</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19595</th>\n",
              "      <td>139</td>\n",
              "      <td>Q0</td>\n",
              "      <td>36</td>\n",
              "      <td>136</td>\n",
              "      <td>0.816049</td>\n",
              "      <td>ADA002</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19596</th>\n",
              "      <td>139</td>\n",
              "      <td>Q0</td>\n",
              "      <td>132</td>\n",
              "      <td>137</td>\n",
              "      <td>0.814769</td>\n",
              "      <td>ADA002</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19597</th>\n",
              "      <td>139</td>\n",
              "      <td>Q0</td>\n",
              "      <td>83</td>\n",
              "      <td>138</td>\n",
              "      <td>0.814735</td>\n",
              "      <td>ADA002</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19598</th>\n",
              "      <td>139</td>\n",
              "      <td>Q0</td>\n",
              "      <td>30</td>\n",
              "      <td>139</td>\n",
              "      <td>0.814437</td>\n",
              "      <td>ADA002</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19599</th>\n",
              "      <td>139</td>\n",
              "      <td>Q0</td>\n",
              "      <td>52</td>\n",
              "      <td>140</td>\n",
              "      <td>0.811439</td>\n",
              "      <td>ADA002</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>19600 rows × 6 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-8e75fbde-07f7-483b-a8a7-531b61ff6e38')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-8e75fbde-07f7-483b-a8a7-531b61ff6e38 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-8e75fbde-07f7-483b-a8a7-531b61ff6e38');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-8d7b1bf3-0dd8-4b0c-a5c6-dea67a3a66a5\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-8d7b1bf3-0dd8-4b0c-a5c6-dea67a3a66a5')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-8d7b1bf3-0dd8-4b0c-a5c6-dea67a3a66a5 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "  <div id=\"id_cc1b24da-aaef-42b2-8e48-2d32242bbbb1\">\n",
              "    <style>\n",
              "      .colab-df-generate {\n",
              "        background-color: #E8F0FE;\n",
              "        border: none;\n",
              "        border-radius: 50%;\n",
              "        cursor: pointer;\n",
              "        display: none;\n",
              "        fill: #1967D2;\n",
              "        height: 32px;\n",
              "        padding: 0 0 0 0;\n",
              "        width: 32px;\n",
              "      }\n",
              "\n",
              "      .colab-df-generate:hover {\n",
              "        background-color: #E2EBFA;\n",
              "        box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "        fill: #174EA6;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate {\n",
              "        background-color: #3B4455;\n",
              "        fill: #D2E3FC;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate:hover {\n",
              "        background-color: #434B5C;\n",
              "        box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "        filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "        fill: #FFFFFF;\n",
              "      }\n",
              "    </style>\n",
              "    <button class=\"colab-df-generate\" onclick=\"generateWithVariable('file_run_ada')\"\n",
              "            title=\"Generate code using this dataframe.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M7,19H8.4L18.45,9,17,7.55,7,17.6ZM5,21V16.75L18.45,3.32a2,2,0,0,1,2.83,0l1.4,1.43a1.91,1.91,0,0,1,.58,1.4,1.91,1.91,0,0,1-.58,1.4L9.25,21ZM18.45,9,17,7.55Zm-12,3A5.31,5.31,0,0,0,4.9,8.1,5.31,5.31,0,0,0,1,6.5,5.31,5.31,0,0,0,4.9,4.9,5.31,5.31,0,0,0,6.5,1,5.31,5.31,0,0,0,8.1,4.9,5.31,5.31,0,0,0,12,6.5,5.46,5.46,0,0,0,6.5,12Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "    <script>\n",
              "      (() => {\n",
              "      const buttonEl =\n",
              "        document.querySelector('#id_cc1b24da-aaef-42b2-8e48-2d32242bbbb1 button.colab-df-generate');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      buttonEl.onclick = () => {\n",
              "        google.colab.notebook.generateWithVariable('file_run_ada');\n",
              "      }\n",
              "      })();\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "file_run_ada",
              "summary": "{\n  \"name\": \"file_run_ada\",\n  \"rows\": 19600,\n  \"fields\": [\n    {\n      \"column\": \"query_id\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 40,\n        \"min\": 0,\n        \"max\": 139,\n        \"num_unique_values\": 140,\n        \"samples\": [\n          108,\n          67,\n          31\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"q0\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          \"Q0\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"doc_id\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 40,\n        \"min\": 0,\n        \"max\": 139,\n        \"num_unique_values\": 140,\n        \"samples\": [\n          60\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"rank\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 40,\n        \"min\": 1,\n        \"max\": 140,\n        \"num_unique_values\": 140,\n        \"samples\": [\n          109\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"score\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.02259630222915543,\n        \"min\": 0.7274358745746111,\n        \"max\": 0.9053969111406552,\n        \"num_unique_values\": 19600,\n        \"samples\": [\n          0.8464337797770978\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"system\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          \"ADA002\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 60
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def make_json_query_bestchunks(file_run, filename, json_mode=\"text\"):\n",
        "    n = [3,5,7,9]\n",
        "    for ix in n:\n",
        "\n",
        "        query_context = {}\n",
        "\n",
        "        for query_id in topics.keys():\n",
        "            hits = file_run[file_run[\"query_id\"] == query_id]\n",
        "            hits = hits.sort_values(by = \"rank\")\n",
        "            if json_mode == \"text\":\n",
        "                query_context[topics.get(query_id)] = [corpus.get(doc) for doc in list(hits[\"doc_id\"].values)[:ix]]\n",
        "            else:\n",
        "                query_context[int(query_id)] = [int(doc_id) for doc_id in list(hits[\"doc_id\"].values)[:ix]]\n",
        "\n",
        "        try:\n",
        "          file_path = f'/content/drive/MyDrive/Artigos/RAG-stuffs/data/data_retriever/{filename}_{ix}_contexts_to_llm.json'\n",
        "          with open(file_path, 'w', encoding='utf-8') as json_file:\n",
        "              json.dump(query_context, json_file, ensure_ascii=False, indent=1)\n",
        "        except:\n",
        "          file_path = f'/content/data_retriever/{filename}_{ix}_contexts_to_llm.json'\n",
        "          with open(file_path, 'w', encoding='utf-8') as json_file:\n",
        "              json.dump(query_context, json_file, ensure_ascii=False, indent=1)"
      ],
      "metadata": {
        "id": "IoSNZs5yJNAK"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!rm -rf data_retriever\n",
        "!mkdir data_retriever"
      ],
      "metadata": {
        "id": "MtFfUZkYJh6J"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "make_json_query_bestchunks(file_run=file_run_ada, filename=\"reranker\")\n",
        "exp_json = \"reranker_3_contexts_to_llm.json\"\n",
        "try:\n",
        "    path_folder = \"/content/drive/MyDrive/Artigos/RAG-stuffs/data/data_retriever/\"\n",
        "    with open(path_folder + exp_json, 'r', encoding='utf-8') as json_file:\n",
        "        loaded_data = json.load(json_file)\n",
        "except:\n",
        "    path_folder = \"/content/data_retriever/\"\n",
        "    with open(path_folder + exp_json, 'r', encoding='utf-8') as json_file:\n",
        "        loaded_data = json.load(json_file)"
      ],
      "metadata": {
        "id": "JdvuZjy_KzAS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!head -n 5 /content/data_retriever/reranker_3_contexts_to_llm.json"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_CAeYmICFg03",
        "outputId": "bc951699-bf19-441a-db19-6ada7bf323a2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{\n",
            " \"Qual é o segredo que o Sr. e a Sra. Dursley temiam que alguém descobrisse?\": [\n",
            "  \"- CAPÍTULO UM - O menino que sobreviveu O Sr. e a Sra. Dursley, da rua dos Alfeneiros, no 4, se orgulhavam de dizer que eram perfeitamente normais, muito bem, obrigado. Eram as últimas pessoas no mundo que se esperaria que se metessem em alguma coisa estranha ou misteriosa, porque simplesmente não compactuavam com esse tipo de bobagem. O Sr. Dursley era diretor de uma firma chamada Grunnings, que fazia perfurações. Era um homem alto e corpulento quase sem pescoço, embora tivesse enormes bigodes. A Sra. Dursley era magra e loura e tinha um pescoço quase duas vezes mais comprido que o normal, o que era muito útil porque ela passava grande parte do tempo espichando-o por cima da cerca do jardim para espiar os vizinhos. Os Dursley tinham um filhinho chamado Dudley, o Duda, e em sua opinião não havia garoto melhor em nenhum lugar do mundo. Os Dursley tinham tudo que queriam, mas tinham também um segredo, e seu maior receio era que alguém o descobrisse. Achavam que não iriam aguentar se alguém descobrisse a existência dos Potter. A Sra. Potter era irmã da Sra. Dursley, mas não se viam havia muitos anos; na realidade a Sra. Dursley fingia que não tinha irmã, porque esta e o marido imprestável eram o que havia de menos parecido possível com os Dursley. Eles estremeciam só de pensar o que os vizinhos iriam dizer se os Potter aparecessem na rua. Os Dursley sabiam que os Potter tinham um filhinho, também, mas nunca o tinham visto. O garoto era mais uma razão para manter os Potter a distância; eles não queriam que Duda se misturasse com uma criança daquelas. Quando o Sr. e a Sra. Dursley acordaram na terça-feira monótona e cinzenta em que a nossa história começa, não havia nada no céu nublado lá fora sugerindo as coisas estranhas e misteriosas que não tardariam a acontecer por todo o país. O Sr. Dursley cantarolava ao escolher a gravata mais sem graça do mundo para ir trabalhar e a Sra. Dursley fofocava alegremente enquanto lutava  para encaixar um Duda aos berros na cadeirinha alta. Nenhum deles reparou em uma coruja parda que passou, batendo as asas, pela janela. Às oito e meia, o Sr. Dursley apanhou a pasta, deu um beijinho no rosto da Sra. Dursley e tentou dar um beijo de despedida em Duda mas não conseguiu, porque na hora Duda estava tendo um acesso de raiva e atirava o cereal nas paredes. - Pestinha - disse rindo contrafeito o Sr. Dursley ao sair de casa. Entrou no carro e deu marcha à ré para sair do estacionamento do número quatro. Foi na esquina da rua que ele notou o primeiro indício de que algo estranho ocorria - um gato lia um mapa. Por um instante o Sr. Dursley não percebeu o que vira - em seguida virou rapidamente a cabeça para dar uma segunda olhada. Havia um gato de listras amarelas sentado na esquina da rua dos Alfeneiros, mas não havia nenhum mapa à vista. Em que estaria pensando naquela hora? Devia ter sido um efeito da luz. Ele piscou e arregalou os olhos para o gato. O gato o encarou. Enquanto virava a esquina e subia a rua, espiou o gato pelo espelho retrovisor. Ele agora estava lendo a placa que dizia rua dos Alfeneiros - não, estava olhando a placa: gatos não podiam ler mapas nem placas. O Sr. Dursley sacudiu a cabeça e tirou o gato do pensamento. Dur\",\n",
            "  \"pando a capa, procurando alguma coisa. Mas parecia ter consciência de que estava sendo vigiado, porque ergueu a cabeça de repente para o gato, que continuava a fixá-lo da outra ponta da rua. Por algum motivo, a visão do gato pareceu diverti-lo. Deu uma risadinha e murmurou: \\\"Eu devia ter imaginado.\\\" Encontrou o que procurava no bolso interior da capa. Parecia um isqueiro de prata. Abriu-o, ergueu-o no ar e o acendeu. O lampião de rua mais próximo apagou-se com um estalido seco. Ele o acendeu de novo - o lampião seguinte piscou e apagou, doze vezes ele acionou o \\\"apagueiro\\\", até que as únicas luzes acesas na rua toda eram dois pontinhos minúsculos ao longe - os olhos do gato que o vigiava. Se alguém espiasse pela janela agora, até a Sra. Dursley, de olhos de contas, não conseguiria ver nada que estava acontecendo na calçada. Dumbledore tornou a guardar o \\\"apagueiro\\\" na capa e saiu caminhando pela rua em direção ao número quatro, onde se sentou no muro ao lado do gato. Não olhou para o bicho, mas, passado algum tempo, dirigiu-se a ele. - Imaginei encontrar a senhora aqui, Profa. Minerva McGonagall. E virou-se para sorrir para o gato, mas este desaparecera. Em vez dele, viu-se sorrindo para uma mulher de aspecto severo que usava óculos de lestes quadradas exatamente do formato das marcas que o gato tinha em volta dos olhos. Ela, também, usava uma capa esmeralda. Trazia os cabelos negros presos num coque apertado. E parecia decididamente irritada. - Como soube que era eu? - perguntou. - Minha cara professora, nunca vi um gato se sentar tão duro. - O senhor estaria duro se tivesse passado o dia todo sentado em um muro de pedra - respondeu a Profa. Minerva. - O dia todo? Quando podia estar comemorando? Devo ter passado por mais de dez festas e banquetes a caminho daqui. A professora fungou aborrecida. - Ah, sim, vi que todos estão comemorando - disse impaciente. - Era de esperar que fossem um pouco mais cautelosos, mas não, até os trouxas notaram que alguma coisa estava acontecendo. Deu no telejornal. - Ela indicou com a  cabeça a sala às escuras dos Dursley. - Eu ouvi... bandos de corujas... estrelas cadentes... Ora, eles não são completamente idiotas. Não podiam deixar de notar alguma coisa. Estrelas cadentes em Kent, aposto que foi coisa do Dédalo Diggle. Ele nunca teve muito juízo. - Você não pode culpá-los - ponderou Dumbledore educadamente. - Temos tido muito pouco o que comemorar nos últimos onze anos. - Sei disso - retrucou a professora mal-humorada. - Mas não é razão para perdermos a cabeça. As pessoas estão sendo completamente descuidadas, saem às ruas em plena luz do dia, sem nem ao menos vestir roupa de trouxa, e espalham boatos. De esguelha, lançou um olhar atento a Dumbledore, como se esperasse que ele dissesse alguma coisa, mas ele continuou calado, por isso ela recomeçou: - Ia ser uma graça se, no próprio dia em que Você-Sabe-Quem parece ter finalmente ido embora, os trouxas descobrissem a nossa existência. Suponho que ele realmente tenha ido embora, não é, Dumbledore? - Parece que não há dúvida. Temos muito o que agradecer. Aceita um sorvete de limão? - Um o quê? - Um sorvete de limão. É uma espécie de doce dos trouxas de que sempre gostei muito. - Não, obrigada - disse a Profa\",\n",
            "  \" deixou escrito naquela carta para ele? Eu estava lá! Eu vi Dumbledore deixar a carta, Dursley! E você escondeu dele todos esses anos? - Escondeu o que de mim? - perguntou Harry, ansioso. - PARE! EU O PROÍBO! - gritou tio Válter em pânico. Tia Petúnia deixou escapar um grito sufocado de horror. - Ah, vão tomar banho, vocês dois - disse Hagrid. - Harry, você é um bruxo. O casebre mergulhou em silêncio. Ouviam-se apenas o mar e o assobio do vento. - Eu sou o quê ? - ofegou Harry - Um bruxo, é claro - repetiu Hagrid, recostando-se no sofá, que gemeu e afundou ainda mais -, e um bruxo de primeira, eu diria, depois que receber um pequeno treino. Com uma mãe e um pai como os seus, o que mais você poderia ser? E acho que já está na hora de ler a sua carta. Harry estendeu a mão finalmente para receber o envelope meio amarelo, endereçado em tinta verde para Sr. H. Potter, O Soalho, Casebre-sobre-o- Rochedo, O Mar. Ele puxou a carta e leu:   ESCOLA DE MAGIA E BRUXARIA DE HOGWARTS   Diretor: Alvo Dumbledore (Ordem de Merlim, Primeira Classe, Grande Feiticeiro, Bruxo Chefe, Cacique Supremo, Confederação Internacional de Bruxos)   Prezado Sr. Potter, Temos o prazer de informar que V. Sa. tem uma vaga na Escola de Magia e Bruxaria de Hogwarts. Estamos anexando uma lista dos livros e equipamentos necessários. O ano letivo começa em 1o de setembro. Aguardamos sua coruja até 31 de  julho, no mais tardar. Atenciosamente, Minerva McGonagall Diretora Substituta   As perguntas explodiam na cabeça de Harry como fogos de artifício, e ele não conseguia decidir o que perguntar primeiro. Passados alguns minutos, gaguejou: - O que querem dizer com \\\"estão aguardando a minha coruja\\\"? - Gárgulas galopantes! Isto me lembra uma coisa - disse Hagrid, batendo a mão na testa com força suficiente para derrubar um cavalo, e de outro bolso interno do casaco tirou uma coruja, uma coruja de verdade, viva, meio arrepiada, uma longa pena e um rolo de pergaminho. Com a língua entre os dentes, ele rabiscou um bilhete que Harry pôde ler de cabeça para baixo:   \\\"Prezado Sr. Dumbledore, Entreguei a carta a Harry. Vou levá-lo amanhã para comprar o material. O tempo está horrível. Espero que o senhor esteja bem. Hagrid.\\\"   Hagrid enrolou o pergaminho, entregou-o à coruja, que o prendeu no bico, depois ele foi até a porta e lançou a ave na tempestade. Quando voltou, sentou-se como se aquilo fosse tão normal quanto pegar o telefone. Harry percebeu que sua boca se abrira e fechou-a rapidamente. - Onde é que eu estava? - disse Hagrid, mas, naquele momento, tio Válter, ainda cor de cera, mas parecendo muito furioso, adiantou-se até a luz da lareira. - Ele não vai - falou. Hagrid resmungou. - Eu gostaria de ver um grande trouxa como você impedi-lo - respondeu. - Um o quê? - perguntou Harry, interessado. - Um trouxa - disse Hagrid -, é como chamamos gente que não é mágica como nós. E você teve o azar de ser criado na família dos maiores trouxas que já vi na vida. - Juramos quando o aceitamos que poríamos um fim nessa bobagem - disse tio Válter -, juramos que erradicaríamos isso nele. Bruxo, francamente! - Você sabia? - perguntou Harry. - Você sabia que\"\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "qa_prompt = \"\"\"Com muita calma, responda a seguinte pergunta dentro do `contexto` que será fornecido abaixo.\n",
        "Utilize somente informações do contexto para responder.\n",
        "\n",
        "Pergunta: ```{pergunta}```\n",
        "\n",
        "\"\"\"\n",
        "\n",
        "get_context = \"\"\"\n",
        "## inicio do contexto {ix}.\n",
        "{contexto}\n",
        "## fim do contexto {ix}.\n",
        "\n",
        "###\n",
        "\"\"\"\n",
        "\n",
        "final_format = \"\"\"\\nO output deve ser somente a resposta.\"\"\"\n",
        "\n",
        "\n",
        "for query, chunks in loaded_data.items():\n",
        "    for chunk in chunks:\n",
        "\n",
        "        chunks_retrieved = ''\n",
        "        for ix, chunk in enumerate(chunks, 1):\n",
        "            chunks_retrieved += get_context.format(contexto=chunk, ix=ix)\n",
        "\n",
        "            if ix== len(chunks):\n",
        "                prompt = qa_prompt.format(pergunta=query) + chunks_retrieved + final_format"
      ],
      "metadata": {
        "id": "AqKCFSUaXgvg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(prompt)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WR2j_acpLyGq",
        "outputId": "79a7e941-2685-4d6e-f233-d4f476b8fdd7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Com muita calma, responda a seguinte pergunta dentro do `contexto` que será fornecido abaixo.\n",
            "Utilize somente informações do contexto para responder.\n",
            "\n",
            "Pergunta: ```Qual é a razão específica pela qual Harry Potter está sorrindo ao se despedir de Hermione e imaginar suas férias com o Duda?```\n",
            "\n",
            "\n",
            "## inicio do contexto 1.\n",
            " - Vejo vocês durante as férias, então. - Espero que você tenha... hã... umas boas férias - disse Hermione, olhando hesitante para tio Válter, espantada que alguém pudesse ser tão desagradável. - Ah, claro que sim - respondeu Harry, e eles ficaram surpresos com o sorriso que se espalhava pelo seu rosto. - Eles não sabem que não podemos fazer bruxarias em casa. Vou me divertir à beça com o Duda este verão...\n",
            "## fim do contexto 1.\n",
            "\n",
            "###\n",
            "\n",
            "## inicio do contexto 2.\n",
            "umo sem se dar conta de que o cachimbo apagara. Hagrid sorria radiante. Logo houve um grande arrastar de cadeiras e no momento seguinte Harry se viu apertando as mãos de todos n\"O Caldeirão Furado. - Dóris Crockford, Sr. Potter, não acredito que finalmente posso conhecê-lo. - Estou tão orgulhosa, Sr. Potter, tão orgulhosa. - Sempre quis apertar sua mão. Estou nas nuvens. - Encantado, Sr. Potter, nem sei lhe dizer o quanto, Diggle é o meu nome, Dédalo Diggle. - Já vi o senhor antes! - disse Harry, e a cartola de Diggle caiu de tanta excitação. - O senhor se curvou para mim uma vez numa loja. - Ele se lembra! - exclamou Dédalo Diggle, olhando todos à volta. - Vocês ouviram isso? Ele se lembra de mim! Harry apertou muitas mãos. Dóris Crockford não parava de voltar para um novo aperto.  Um rapaz pálido adiantou-se, muito nervoso. Um olho trêmulo. - Prof. Quirrell! - disse Hagrid. - Harry, o Prof. Quirrell vai ser um dos seus professores em Hogwarts. - P-P-Potter - gaguejou o Prof. Quirrell, apertando a mão de Harry -, n-n-em sei d-d-dizer que p-p-p-prazer enorme é c-c-conhecê-lo. - Que tipo de mágica o senhor ensina, Prof. Quirrell? - D-d-defesa C-c-ontra as Art-t-tes das T-t-trevas - murmurou o Prof. Quirrell, como se preferisse não pensar no assunto. - N-n-não que você p-p- precise, hein, Potter? - Ele riu nervoso. - V-v-você veio c-c-comprar o material, suponho? Tenho que c-c-comprar um livro n-n-novo sobre vampiros. - Parecia aterrorizado só de pensar. Mas os outros não queriam deixar o Prof. Quirrell ficar com Harry só para ele. Levou bem uns dez minutos para o menino se livrar de todos. Finalmente, Hagrid conseguiu se fazer ouvir naquela balbúrdia. - Precisamos nos apressar. Temos muitas compras a fazer. Vamos, Harry. Dóris Crockford apertou a mão de Harry uma última vez e eles passaram pelo bar e saíram num pequeno pátio murado, onde não havia nada exceto uma lata de lixo e um pouco de mato. Hagrid sorriu para Harry. - Eu lhe falei, não foi? Falei que você era famoso. Até o professor Quirrell ficou tremendo de emoção de o conhecer, mas, em geral, ele está sempre tremendo. - Ele é sempre tão nervoso? - Ah, é. Coitado. Uma cabeça brilhante. Foi bem enquanto estudou em livros, mas quando tirou um ano para aprender na prática... Dizem que encontrou vampiros na Floresta Negra e teve um problema feio com uma feiticeira, nunca mais foi o mesmo. Tem pavor dos alunos, tem pavor da matéria que ensina, agora, cadê o meu guarda-chuva? Vampiros? Feiticeiras? A cabeça de Harry estava girando. Entrementes, Hagrid contava tijolos na parede por cima da lata de lixo. - Três para cima... dois para o lado... - murmurou. - Certo, chegue para trás, Harry. Ele bateu na parede três vezes com a ponta do guarda-chuva. E o tijolo que tocou estremeceu, então torceu-se. No meio apareceu um buraquinho, que se foi alargando cada vez mais. Um segundo depois se viram diante de um arco bastante grande até para Hagrid, um arco que abria para uma rua de pedras irregulares, serpeava e desaparecia de vista. - Bem-vindo - disse Hagrid - ao Beco Diagonal. Ele riu do espanto de Harry. A\n",
            "## fim do contexto 2.\n",
            "\n",
            "###\n",
            "\n",
            "## inicio do contexto 3.\n",
            "ã dela produziu a sua cicatriz. Harry engoliu em seco. - É, tinha trinta e quatro centímetros. Puxa. É realmente curioso como essas coisas acontecem. A varinha escolhe o bruxo, lembre-se... Acho que podemos esperar grandes feitos do senhor, Sr. Potter... Afinal, Aquele-Que-Não-Se-Deve- Nomear realizou grandes feitos, terríveis, sim, mas grandes. Harry estremeceu. Não tinha muita certeza se gostava do Sr. Olivaras. Pagou sete galeões pela varinha e o Sr. Olivaras curvou-se à saída deles. O sol de fim de tarde quase chegara ao horizonte quando Harry e Hagrid refizeram o caminho para sair do Beco Diagonal, atravessar a parede e passar novamente pelo Caldeirão Furado, agora vazio. Harry não disse uma palavra enquanto caminhavam pela rua; nem ao menos reparou quantas pessoas se boquiabriam para eles no metrô, carregados que estavam com todos aqueles pacotes de formatos esquisitos, a coruja branca adormecida no colo de Harry. Subiram a escada rolante para a estação de Paddington; Harry só percebeu onde estavam quando Hagrid bateu em seu ombro.  - Temos tempo para comer alguma coisa antes de o trem sair - falou. Comprou um hambúrguer para Harry e se sentaram em bancos de plástico para comê-los. Harry não parava de olhar a toda volta. Por alguma razão tudo parecia tão estranho. - Você está bem, Harry? Está muito calado - comentou Hagrid. Harry não tinha muita certeza de poder explicar. Tivera o melhor aniversário de sua vida, porém... e mastigava o hambúrguer, tentando encontrar as palavras. - Todo o mundo acha que sou especial - disse finalmente. - Todas aquelas pessoas no Caldeirão Furado, o Prof. Quirrell, o Sr. Olivaras... mas eu não conheço nadinha de mágica. Como podem esperar grandes feitos de mim? Sou famoso e nem ao menos me lembro o porquê. Não sei o que aconteceu quando Vol... desculpe... quero dizer, na noite que meus pais morreram. Hagrid se debruçou sobre a mesa. Por trás da barba e das sobrancelhas desgrenhadas tinha um sorriso bondoso. - Não se preocupe, Harry. Você vai aprender bem depressa. Todos começam pelo começo em Hogwarts, você vai se dar bem. Seja você mesmo. Sei que é difícil. Você vai ser discriminado e isso é muito duro. Mas vai se divertir a valer em Hogwarts. Eu me diverti; e ainda me divirto, para dizer a verdade. Hagrid ajudou Harry a embarcar no trem que o levaria de volta aos Dursley, então lhe entregou um envelope. - A sua passagem para Hogwarts. Primeiro de setembro, na estação de King\"s Cross, está tudo na passagem. Qualquer problema com os Dursley, me mande uma carta pela coruja, ela saberá onde me encontrar... Vejo você em breve, Harry. O trem parou na estação. Harry queria ficar espiando Hagrid até ele desaparecer de vista; levantou-se, espremeu o nariz contra o vidro da janela, mas quando piscou os olhos Hagrid tinha desaparecido.  - CAPÍTULO SEIS - O embarque na plataforma nove e meia O último mês de Harry na casa dos Dursley não foi nada divertido. É verdade que Duda agora estava tão apavorado com Harry que não queria nem ficar no mesmo aposento com ele, e tia Petúnia e tio Válter não trancaram Harry no armário nem o obrigaram a fazer nada, tampouco gritaram com ele - na verdade, sequer falaram com ele. Meio aterrorizados, meio furiosos, agiam como se a cadeira em que Harry se sentasse estivesse vazia. Embora isso fosse sob muitos aspectos um progresso, tornou-se um tanto deprimente\n",
            "## fim do contexto 3.\n",
            "\n",
            "###\n",
            "\n",
            "O output deve ser somente a resposta.\n"
          ]
        }
      ]
    }
  ],
  "metadata": {
    "colab": {
      "machine_shape": "hm",
      "provenance": [],
      "gpuClass": "premium",
      "gpuType": "V100",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "nbformat": 4,
  "nbformat_minor": 0
}