RAG_paper.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/finardi/tutos/blob/master/RAG_paper.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YNbFOEtDTnKL"
      },
      "source": [
        "## Mount Google Drive"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jydgbyHSbtIl",
        "outputId": "1585b66f-3010-4e6e-a3be-37458301fa3e"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Install Libs"
      ],
      "metadata": {
        "id": "bdau-U9CHM8B"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XPF1_OOrRYaj",
        "outputId": "f951bc8f-4836-4212-a7fe-772a076d405d"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m17.6/17.6 MB\u001b[0m \u001b[31m72.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m142.1/142.1 MB\u001b[0m \u001b[31m8.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.5/1.5 MB\u001b[0m \u001b[31m71.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m70.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m188.7/188.7 kB\u001b[0m \u001b[31m22.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.4/6.4 MB\u001b[0m \u001b[31m95.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m224.7/224.7 kB\u001b[0m \u001b[31m19.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.0/2.0 MB\u001b[0m \u001b[31m81.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m46.0/46.0 kB\u001b[0m \u001b[31m6.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.9/75.9 kB\u001b[0m \u001b[31m11.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m76.9/76.9 kB\u001b[0m \u001b[31m11.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m9.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m86.8/86.8 kB\u001b[0m \u001b[31m12.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Building wheel for nmslib (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "llmx 0.0.15a0 requires cohere, which is not installed.\n",
            "tensorflow-probability 0.22.0 requires typing-extensions<4.6.0, but you have typing-extensions 4.9.0 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m53.4/53.4 kB\u001b[0m \u001b[31m2.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: tiktoken in /usr/local/lib/python3.10/dist-packages (0.5.2)\n",
            "Requirement already satisfied: regex>=2022.1.18 in /usr/local/lib/python3.10/dist-packages (from tiktoken) (2023.6.3)\n",
            "Requirement already satisfied: requests>=2.26.0 in /usr/local/lib/python3.10/dist-packages (from tiktoken) (2.31.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.26.0->tiktoken) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.26.0->tiktoken) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.26.0->tiktoken) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.26.0->tiktoken) (2023.11.17)\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": []
          },
          "metadata": {},
          "execution_count": 2
        }
      ],
      "source": [
        "%%shell\n",
        "pip install faiss-cpu -q\n",
        "pip install pyserini -q\n",
        "pip install ftfy -q\n",
        "pip install tiktoken"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qUABwgO1Rbtd",
        "outputId": "c527ddd3-ccb0-426c-9cdb-c8bd876dcab5"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Extracting templates from packages: 100%\n",
            "Selecting previously unselected package libapache-pom-java.\n",
            "(Reading database ... 121654 files and directories currently installed.)\n",
            "Preparing to unpack .../00-libapache-pom-java_18-1_all.deb ...\n",
            "Unpacking libapache-pom-java (18-1) ...\n",
            "Selecting previously unselected package libatinject-jsr330-api-java.\n",
            "Preparing to unpack .../01-libatinject-jsr330-api-java_1.0+ds1-5_all.deb ...\n",
            "Unpacking libatinject-jsr330-api-java (1.0+ds1-5) ...\n",
            "Selecting previously unselected package libgeronimo-interceptor-3.0-spec-java.\n",
            "Preparing to unpack .../02-libgeronimo-interceptor-3.0-spec-java_1.0.1-4fakesync_all.deb ...\n",
            "Unpacking libgeronimo-interceptor-3.0-spec-java (1.0.1-4fakesync) ...\n",
            "Selecting previously unselected package libcdi-api-java.\n",
            "Preparing to unpack .../03-libcdi-api-java_1.2-3_all.deb ...\n",
            "Unpacking libcdi-api-java (1.2-3) ...\n",
            "Selecting previously unselected package libcommons-cli-java.\n",
            "Preparing to unpack .../04-libcommons-cli-java_1.4-2_all.deb ...\n",
            "Unpacking libcommons-cli-java (1.4-2) ...\n",
            "Selecting previously unselected package libcommons-parent-java.\n",
            "Preparing to unpack .../05-libcommons-parent-java_43-1_all.deb ...\n",
            "Unpacking libcommons-parent-java (43-1) ...\n",
            "Selecting previously unselected package libcommons-io-java.\n",
            "Preparing to unpack .../06-libcommons-io-java_2.11.0-2_all.deb ...\n",
            "Unpacking libcommons-io-java (2.11.0-2) ...\n",
            "Selecting previously unselected package libcommons-lang3-java.\n",
            "Preparing to unpack .../07-libcommons-lang3-java_3.11-1_all.deb ...\n",
            "Unpacking libcommons-lang3-java (3.11-1) ...\n",
            "Selecting previously unselected package libgeronimo-annotation-1.3-spec-java.\n",
            "Preparing to unpack .../08-libgeronimo-annotation-1.3-spec-java_1.3-1_all.deb ...\n",
            "Unpacking libgeronimo-annotation-1.3-spec-java (1.3-1) ...\n",
            "Selecting previously unselected package libjsr305-java.\n",
            "Preparing to unpack .../09-libjsr305-java_0.1~+svn49-11_all.deb ...\n",
            "Unpacking libjsr305-java (0.1~+svn49-11) ...\n",
            "Selecting previously unselected package libguava-java.\n",
            "Preparing to unpack .../10-libguava-java_29.0-6_all.deb ...\n",
            "Unpacking libguava-java (29.0-6) ...\n",
            "Selecting previously unselected package libaopalliance-java.\n",
            "Preparing to unpack .../11-libaopalliance-java_20070526-6_all.deb ...\n",
            "Unpacking libaopalliance-java (20070526-6) ...\n",
            "Selecting previously unselected package libguice-java.\n",
            "Preparing to unpack .../12-libguice-java_4.2.3-2_all.deb ...\n",
            "Unpacking libguice-java (4.2.3-2) ...\n",
            "Selecting previously unselected package libhawtjni-runtime-java.\n",
            "Preparing to unpack .../13-libhawtjni-runtime-java_1.17-1_all.deb ...\n",
            "Unpacking libhawtjni-runtime-java (1.17-1) ...\n",
            "Selecting previously unselected package libjansi-native-java.\n",
            "Preparing to unpack .../14-libjansi-native-java_1.8-1_all.deb ...\n",
            "Unpacking libjansi-native-java (1.8-1) ...\n",
            "Selecting previously unselected package libjansi-java.\n",
            "Preparing to unpack .../15-libjansi-java_1.18-1_all.deb ...\n",
            "Unpacking libjansi-java (1.18-1) ...\n",
            "Selecting previously unselected package libmaven-parent-java.\n",
            "Preparing to unpack .../16-libmaven-parent-java_31-2_all.deb ...\n",
            "Unpacking libmaven-parent-java (31-2) ...\n",
            "Selecting previously unselected package libplexus-utils2-java.\n",
            "Preparing to unpack .../17-libplexus-utils2-java_3.3.0-1_all.deb ...\n",
            "Unpacking libplexus-utils2-java (3.3.0-1) ...\n",
            "Selecting previously unselected package libwagon-provider-api-java.\n",
            "Preparing to unpack .../18-libwagon-provider-api-java_3.3.4-1_all.deb ...\n",
            "Unpacking libwagon-provider-api-java (3.3.4-1) ...\n",
            "Selecting previously unselected package libmaven-resolver-java.\n",
            "Preparing to unpack .../19-libmaven-resolver-java_1.4.2-3build1_all.deb ...\n",
            "Unpacking libmaven-resolver-java (1.4.2-3build1) ...\n",
            "Selecting previously unselected package libmaven-shared-utils-java.\n",
            "Preparing to unpack .../20-libmaven-shared-utils-java_3.3.0-1_all.deb ...\n",
            "Unpacking libmaven-shared-utils-java (3.3.0-1) ...\n",
            "Selecting previously unselected package libplexus-cipher-java.\n",
            "Preparing to unpack .../21-libplexus-cipher-java_1.8-2_all.deb ...\n",
            "Unpacking libplexus-cipher-java (1.8-2) ...\n",
            "Selecting previously unselected package libplexus-classworlds-java.\n",
            "Preparing to unpack .../22-libplexus-classworlds-java_2.6.0-1_all.deb ...\n",
            "Unpacking libplexus-classworlds-java (2.6.0-1) ...\n",
            "Selecting previously unselected package libplexus-component-annotations-java.\n",
            "Preparing to unpack .../23-libplexus-component-annotations-java_2.1.0-1_all.deb ...\n",
            "Unpacking libplexus-component-annotations-java (2.1.0-1) ...\n",
            "Selecting previously unselected package libplexus-interpolation-java.\n",
            "Preparing to unpack .../24-libplexus-interpolation-java_1.26-1_all.deb ...\n",
            "Unpacking libplexus-interpolation-java (1.26-1) ...\n",
            "Selecting previously unselected package libplexus-sec-dispatcher-java.\n",
            "Preparing to unpack .../25-libplexus-sec-dispatcher-java_1.4-4_all.deb ...\n",
            "Unpacking libplexus-sec-dispatcher-java (1.4-4) ...\n",
            "Selecting previously unselected package libslf4j-java.\n",
            "Preparing to unpack .../26-libslf4j-java_1.7.32-1_all.deb ...\n",
            "Unpacking libslf4j-java (1.7.32-1) ...\n",
            "Selecting previously unselected package libsisu-inject-java.\n",
            "Preparing to unpack .../27-libsisu-inject-java_0.3.4-2_all.deb ...\n",
            "Unpacking libsisu-inject-java (0.3.4-2) ...\n",
            "Selecting previously unselected package libsisu-plexus-java.\n",
            "Preparing to unpack .../28-libsisu-plexus-java_0.3.4-3_all.deb ...\n",
            "Unpacking libsisu-plexus-java (0.3.4-3) ...\n",
            "Selecting previously unselected package libmaven3-core-java.\n",
            "Preparing to unpack .../29-libmaven3-core-java_3.6.3-5_all.deb ...\n",
            "Unpacking libmaven3-core-java (3.6.3-5) ...\n",
            "Selecting previously unselected package libwagon-file-java.\n",
            "Preparing to unpack .../30-libwagon-file-java_3.3.4-1_all.deb ...\n",
            "Unpacking libwagon-file-java (3.3.4-1) ...\n",
            "Selecting previously unselected package libwagon-http-shaded-java.\n",
            "Preparing to unpack .../31-libwagon-http-shaded-java_3.3.4-1_all.deb ...\n",
            "Unpacking libwagon-http-shaded-java (3.3.4-1) ...\n",
            "Selecting previously unselected package maven.\n",
            "Preparing to unpack .../32-maven_3.6.3-5_all.deb ...\n",
            "Unpacking maven (3.6.3-5) ...\n",
            "Setting up libslf4j-java (1.7.32-1) ...\n",
            "Setting up libplexus-utils2-java (3.3.0-1) ...\n",
            "Setting up libplexus-classworlds-java (2.6.0-1) ...\n",
            "Setting up libjsr305-java (0.1~+svn49-11) ...\n",
            "Setting up libaopalliance-java (20070526-6) ...\n",
            "Setting up libcommons-cli-java (1.4-2) ...\n",
            "Setting up libplexus-component-annotations-java (2.1.0-1) ...\n",
            "Setting up libplexus-cipher-java (1.8-2) ...\n",
            "Setting up libgeronimo-annotation-1.3-spec-java (1.3-1) ...\n",
            "Setting up libgeronimo-interceptor-3.0-spec-java (1.0.1-4fakesync) ...\n",
            "Setting up libapache-pom-java (18-1) ...\n",
            "Setting up libatinject-jsr330-api-java (1.0+ds1-5) ...\n",
            "Setting up libplexus-interpolation-java (1.26-1) ...\n",
            "Setting up libplexus-sec-dispatcher-java (1.4-4) ...\n",
            "Setting up libwagon-http-shaded-java (3.3.4-1) ...\n",
            "Setting up libcdi-api-java (1.2-3) ...\n",
            "Setting up libhawtjni-runtime-java (1.17-1) ...\n",
            "Setting up libwagon-provider-api-java (3.3.4-1) ...\n",
            "Setting up libmaven-parent-java (31-2) ...\n",
            "Setting up libcommons-parent-java (43-1) ...\n",
            "Setting up libsisu-inject-java (0.3.4-2) ...\n",
            "Setting up libsisu-plexus-java (0.3.4-3) ...\n",
            "Setting up libmaven-resolver-java (1.4.2-3build1) ...\n",
            "Setting up libguava-java (29.0-6) ...\n",
            "Setting up libcommons-lang3-java (3.11-1) ...\n",
            "Setting up libjansi-native-java (1.8-1) ...\n",
            "Setting up libwagon-file-java (3.3.4-1) ...\n",
            "Setting up libcommons-io-java (2.11.0-2) ...\n",
            "Setting up libguice-java (4.2.3-2) ...\n",
            "Setting up libjansi-java (1.18-1) ...\n",
            "Setting up libmaven-shared-utils-java (3.3.0-1) ...\n",
            "Setting up libmaven3-core-java (3.6.3-5) ...\n",
            "Setting up maven (3.6.3-5) ...\n",
            "update-alternatives: using /usr/share/maven/bin/mvn to provide /usr/bin/mvn (mvn) in auto mode\n",
            "Cloning into 'pyserini'...\n",
            "remote: Enumerating objects: 8663, done.\u001b[K\n",
            "remote: Counting objects: 100% (3597/3597), done.\u001b[K\n",
            "remote: Compressing objects: 100% (586/586), done.\u001b[K\n",
            "remote: Total 8663 (delta 3229), reused 3203 (delta 3009), pack-reused 5066\u001b[K\n",
            "Receiving objects: 100% (8663/8663), 7.17 MiB | 28.23 MiB/s, done.\n",
            "Resolving deltas: 100% (6399/6399), done.\n",
            "Submodule 'tools' (https://github.com/castorini/anserini-tools.git) registered for path 'tools'\n",
            "Cloning into '/content/pyserini/tools'...\n",
            "remote: Enumerating objects: 868, done.        \n",
            "remote: Counting objects: 100% (625/625), done.        \n",
            "remote: Compressing objects: 100% (546/546), done.        \n",
            "remote: Total 868 (delta 114), reused 575 (delta 78), pack-reused 243        \n",
            "Receiving objects: 100% (868/868), 213.78 MiB | 27.20 MiB/s, done.\n",
            "Resolving deltas: 100% (198/198), done.\n",
            "Submodule path 'tools': checked out '7b84f773225b5973b4533dfa0aa18653409a6146'\n",
            "trec_eval.9.0.4/\n",
            "trec_eval.9.0.4/m_prefs_pair.c\n",
            "trec_eval.9.0.4/m_ndcg_p.c\n",
            "trec_eval.9.0.4/m_infap.c\n",
            "trec_eval.9.0.4/m_num_q.c\n",
            "trec_eval.9.0.4/m_iprec_at_recall.c\n",
            "trec_eval.9.0.4/form_prefs_counts.c\n",
            "trec_eval.9.0.4/m_prefs_num_prefs_ful_ret.c\n",
            "trec_eval.9.0.4/utility_pool.c\n",
            "trec_eval.9.0.4/m_binG.c\n",
            "trec_eval.9.0.4/meas_avg.c\n",
            "trec_eval.9.0.4/m_gm_bpref.c\n",
            "trec_eval.9.0.4/m_runid.c\n",
            "trec_eval.9.0.4/m_bpref.c\n",
            "trec_eval.9.0.4/m_gm_map.c\n",
            "trec_eval.9.0.4/trec_eval.h\n",
            "trec_eval.9.0.4/m_yaap.c\n",
            "trec_eval.9.0.4/m_relstring.c\n",
            "trec_eval.9.0.4/m_Rprec.c\n",
            "trec_eval.9.0.4/m_prefs_avgjg.c\n",
            "trec_eval.9.0.4/m_success.c\n",
            "trec_eval.9.0.4/m_ndcg.c\n",
            "trec_eval.9.0.4/functions.h\n",
            "trec_eval.9.0.4/m_P_avgjg.c\n",
            "trec_eval.9.0.4/test/\n",
            "trec_eval.9.0.4/test/qrels.rel_level\n",
            "trec_eval.9.0.4/test/results.test\n",
            "trec_eval.9.0.4/test/qrels.test\n",
            "trec_eval.9.0.4/test/out.test.qrels_jg\n",
            "trec_eval.9.0.4/test/out.test.meas_params\n",
            "trec_eval.9.0.4/test/out.test.a\n",
            "trec_eval.9.0.4/test/out.test.prefs\n",
            "trec_eval.9.0.4/test/out.test.aqcM\n",
            "trec_eval.9.0.4/test/out.test.aql\n",
            "trec_eval.9.0.4/test/prefs.test\n",
            "trec_eval.9.0.4/test/out.test\n",
            "trec_eval.9.0.4/test/out.test.aq\n",
            "trec_eval.9.0.4/test/out.test.aqc\n",
            "trec_eval.9.0.4/test/out.test.qrels_prefs\n",
            "trec_eval.9.0.4/test/zscores_file\n",
            "trec_eval.9.0.4/test/qrels.123\n",
            "trec_eval.9.0.4/test/out.test.aqZ\n",
            "trec_eval.9.0.4/test/results.trunc\n",
            "trec_eval.9.0.4/test/prefs.results.test\n",
            "trec_eval.9.0.4/test/prefs.rank20\n",
            "trec_eval.9.0.4/m_11pt_avg.c\n",
            "trec_eval.9.0.4/m_G.c\n",
            "trec_eval.9.0.4/m_num_rel.c\n",
            "trec_eval.9.0.4/m_map_cut.c\n",
            "trec_eval.9.0.4/m_prefs_avgjg_ret.c\n",
            "trec_eval.9.0.4/m_Rprec_mult.c\n",
            "trec_eval.9.0.4/Makefile\n",
            "trec_eval.9.0.4/m_map_avgjg.c\n",
            "trec_eval.9.0.4/get_qrels_prefs.c\n",
            "trec_eval.9.0.4/README\n",
            "trec_eval.9.0.4/m_set_rel_P.c\n",
            "trec_eval.9.0.4/sysfunc.h\n",
            "trec_eval.9.0.4/m_prefs_pair_ret.c\n",
            "trec_eval.9.0.4/convert_zscores.c\n",
            "trec_eval.9.0.4/m_ndcg_cut.c\n",
            "trec_eval.9.0.4/m_prefs_pair_imp.c\n",
            "trec_eval.9.0.4/meas_print_single.c\n",
            "trec_eval.9.0.4/meas_print_final.c\n",
            "trec_eval.9.0.4/trec_eval.c\n",
            "trec_eval.9.0.4/m_num_ret.c\n",
            "trec_eval.9.0.4/get_prefs.c\n",
            "trec_eval.9.0.4/m_P.c\n",
            "trec_eval.9.0.4/get_qrels_jg.c\n",
            "trec_eval.9.0.4/m_rel_P.c\n",
            "trec_eval.9.0.4/meas_acc.c\n",
            "trec_eval.9.0.4/m_prefs_simp.c\n",
            "trec_eval.9.0.4/m_recall.c\n",
            "trec_eval.9.0.4/trec_format.h\n",
            "trec_eval.9.0.4/m_ndcg_rel.c\n",
            "trec_eval.9.0.4/m_num_nonrel_judged_ret.c\n",
            "trec_eval.9.0.4/formats.c\n",
            "trec_eval.9.0.4/bpref_bug\n",
            "trec_eval.9.0.4/README.windows.md\n",
            "trec_eval.9.0.4/m_prefs_num_prefs_ful.c\n",
            "trec_eval.9.0.4/m_set_map.c\n",
            "trec_eval.9.0.4/get_qrels.c\n",
            "trec_eval.9.0.4/m_set_F.c\n",
            "trec_eval.9.0.4/measures.c\n",
            "trec_eval.9.0.4/common.h\n",
            "trec_eval.9.0.4/meas_init.c\n",
            "trec_eval.9.0.4/m_recip_rank.c\n",
            "trec_eval.9.0.4/m_set_recall.c\n",
            "trec_eval.9.0.4/get_trec_results.c\n",
            "trec_eval.9.0.4/m_prefs_simp_ret.c\n",
            "trec_eval.9.0.4/m_num_rel_ret.c\n",
            "trec_eval.9.0.4/m_map.c\n",
            "trec_eval.9.0.4/m_utility.c\n",
            "trec_eval.9.0.4/form_res_rels.c\n",
            "trec_eval.9.0.4/form_res_rels_jg.c\n",
            "trec_eval.9.0.4/m_prefs_simp_imp.c\n",
            "trec_eval.9.0.4/m_prefs_num_prefs_poss.c\n",
            "trec_eval.9.0.4/m_prefs_avgjg_Rnonrel.c\n",
            "trec_eval.9.0.4/m_prefs_avgjg_Rnonrel_ret.c\n",
            "trec_eval.9.0.4/m_set_P.c\n",
            "trec_eval.9.0.4/m_prefs_avgjg_imp.c\n",
            "trec_eval.9.0.4/m_Rndcg.c\n",
            "trec_eval.9.0.4/CHANGELOG\n",
            "trec_eval.9.0.4/m_Rprec_mult_avgjg.c\n",
            "trec_eval.9.0.4/get_zscores.c\n",
            "gcc -g -I.  -Wall -DVERSIONID=\\\"9.0.4\\\"  -o trec_eval trec_eval.c formats.c meas_init.c meas_acc.c meas_avg.c meas_print_single.c meas_print_final.c get_qrels.c get_trec_results.c get_prefs.c get_qrels_prefs.c get_qrels_jg.c form_res_rels.c form_res_rels_jg.c form_prefs_counts.c utility_pool.c get_zscores.c convert_zscores.c measures.c  m_map.c m_P.c m_num_q.c m_num_ret.c m_num_rel.c m_num_rel_ret.c m_gm_map.c m_Rprec.c m_recip_rank.c m_bpref.c m_iprec_at_recall.c m_recall.c m_Rprec_mult.c m_utility.c m_11pt_avg.c m_ndcg.c m_ndcg_cut.c m_Rndcg.c m_ndcg_rel.c m_binG.c m_G.c m_rel_P.c m_success.c m_infap.c m_map_cut.c m_gm_bpref.c m_runid.c m_relstring.c m_set_P.c m_set_recall.c m_set_rel_P.c m_set_map.c m_set_F.c m_num_nonrel_judged_ret.c m_prefs_num_prefs_poss.c m_prefs_num_prefs_ful.c m_prefs_num_prefs_ful_ret.c m_prefs_simp.c m_prefs_pair.c m_prefs_avgjg.c m_prefs_avgjg_Rnonrel.c m_prefs_simp_ret.c m_prefs_pair_ret.c m_prefs_avgjg_ret.c m_prefs_avgjg_Rnonrel_ret.c m_prefs_simp_imp.c m_prefs_pair_imp.c m_prefs_avgjg_imp.c m_map_avgjg.c m_Rprec_mult_avgjg.c m_P_avgjg.c m_yaap.c -lm\n",
            "gcc -o ndeval ndeval.c -lm\n",
            "\u001b[01m\u001b[Kndeval.c:\u001b[m\u001b[K In function ‘\u001b[01m\u001b[Kmain\u001b[m\u001b[K’:\n",
            "\u001b[01m\u001b[Kndeval.c:1435:9:\u001b[m\u001b[K \u001b[01;35m\u001b[Kwarning: \u001b[m\u001b[Kformat not a string literal and no format arguments [\u001b[01;35m\u001b[K\u001b]8;;https://gcc.gnu.org/onlinedocs/gcc/Warning-Options.html#index-Wformat-security\u0007-Wformat-security\u001b]8;;\u0007\u001b[m\u001b[K]\n",
            " 1435 |         \u001b[01;35m\u001b[Kprintf\u001b[m\u001b[K (helpText);\n",
            "      |         \u001b[01;35m\u001b[K^~~~~~\u001b[m\u001b[K\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": []
          },
          "metadata": {},
          "execution_count": 3
        }
      ],
      "source": [
        "%%shell\n",
        "apt-get install maven -qq\n",
        "git clone --recurse-submodules https://github.com/castorini/pyserini.git\n",
        "cd pyserini\n",
        "cd tools/eval && tar xvfz trec_eval.9.0.4.tar.gz && cd trec_eval.9.0.4 && make && cd ../../..\n",
        "cd tools/eval/ndeval && make && cd ../../.."
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Import Libs"
      ],
      "metadata": {
        "id": "1AcoIPKpHQVO"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "aEsUFat2ouIw"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "import re\n",
        "import ftfy\n",
        "import math\n",
        "import json\n",
        "import torch\n",
        "import shutil\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import collections\n",
        "import tiktoken"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from tqdm import tqdm\n",
        "from time import time\n",
        "from IPython.display import display\n",
        "from collections import defaultdict, Counter\n",
        "from pyserini.search.lucene import LuceneSearcher"
      ],
      "metadata": {
        "id": "7g2zNbRRHS3-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import locale\n",
        "locale.getpreferredencoding = lambda: \"UTF-8\""
      ],
      "metadata": {
        "id": "Y2xZAC-x4X_w"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9nabsht5JysO"
      },
      "source": [
        "## Dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JEKaGqglH_FJ"
      },
      "outputs": [],
      "source": [
        "# define data path\n",
        "path = f\"/content/drive/MyDrive/22h/RAG/data/\""
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# clear data\n",
        "!rm -rf \"topics.tsv\" \"chunk.tsv\" \"qrels.tsv\"\n",
        "\n",
        "# select data to run\n",
        "data_selector = \"all\"\n",
        "\n",
        "# data selector\n",
        "if data_selector == \"train\":\n",
        "    print(f\"Downloading '{data_selector}' data\")\n",
        "    shutil.copyfile(os.path.join(path, \"topics_train.tsv\"), \"/content/topics.tsv\")\n",
        "    shutil.copyfile(os.path.join(path, \"corpus_chunk_train.tsv\"), \"/content/chunk.tsv\")\n",
        "    shutil.copyfile(os.path.join(path, \"qrels_test.tsv\"), \"/content/qrels.tsv\") # qrels_train == qrels_test\n",
        "elif data_selector == \"test\":\n",
        "    print(f\"Downloading '{data_selector}' data\")\n",
        "    shutil.copyfile(os.path.join(path, \"topics_test.tsv\"), \"/content/topics.tsv\")\n",
        "    shutil.copyfile(os.path.join(path, \"corpus_chunk_test.tsv\"), \"/content/chunk.tsv\")\n",
        "    shutil.copyfile(os.path.join(path, \"qrels_test.tsv\"), \"/content/qrels.tsv\")\n",
        "elif data_selector == \"all\":\n",
        "    print(f\"Downloading '{data_selector}' data\")\n",
        "    shutil.copyfile(os.path.join(path, \"topics.tsv\"), \"/content/topics.tsv\")\n",
        "    shutil.copyfile(os.path.join(path, \"corpus_chunk.tsv\"), \"/content/chunk.tsv\")\n",
        "    shutil.copyfile(os.path.join(path, \"qrels.tsv\"), \"/content/qrels.tsv\")\n",
        "else:\n",
        "    print(\"Select valid data\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aLunKMj3hF4O",
        "outputId": "2723b01a-21ae-4f1a-db18-d3019527374d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading 'all' data\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SJVYUQUELbYm",
        "outputId": "c01bef39-a8f9-4926-ccf6-c4310e414788"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Reading file on /content/topics.tsv: 140it [00:00, 8055.24it/s]\n"
          ]
        }
      ],
      "source": [
        "# load topics\n",
        "input_tsv = f'/content/topics.tsv'\n",
        "topics = {}\n",
        "with open(input_tsv,'r') as f_in:\n",
        "    for line in tqdm(f_in, desc=f'Reading file on {input_tsv}'):\n",
        "        id, text = line.strip().split('\\t')\n",
        "        topics[int(id)] = ftfy.fix_text(\" \".join(text.replace(\"\\n\", \" \").replace(\"\\t\", \" \").strip().split()))"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "list(topics.items())[:3]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nVKfInBOImAm",
        "outputId": "5584c831-3949-4657-9c49-0e8ba66b861b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[(0,\n",
              "  'Qual é o segredo que o Sr. e a Sra. Dursley temiam que alguém descobrisse?'),\n",
              " (1,\n",
              "  'Qual era a cor da capa que o Sr. Dursley viu um homem mais velho do que ele usando, que o fez pensar que poderia ser uma promoção boba?'),\n",
              " (2,\n",
              "  'Qual foi a reação do Sr. Dursley ao ser abraçado por um estranho que usava uma capa roxa e o chamou de trouxa?')]"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "y_WWJYmsUfLd",
        "outputId": "19d8a8f0-ccdc-451a-fe0c-7ba1e92eb5b5"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Reading file on /content/chunk.tsv: 140it [00:00, 438.77it/s]\n"
          ]
        }
      ],
      "source": [
        "# load chunks\n",
        "input_tsv = f'/content/chunk.tsv'\n",
        "corpus = {}\n",
        "with open(input_tsv,'r') as f_in:\n",
        "    for line in tqdm(f_in, desc=f'Reading file on {input_tsv}'):\n",
        "        id, text = line.strip().split('\\t')\n",
        "        corpus[int(id)] = ftfy.fix_text(\" \".join(text.replace(\"\\n\", \" \").replace(\"\\t\", \" \").strip().split()))"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "list(corpus.items())[:3]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Tuw4KzQ5Iq7N",
        "outputId": "36628a8f-2980-4c21-c5f6-3cc0eb34b612"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[(0,\n",
              "  '- CAPÍTULO UM - O menino que sobreviveu O Sr. e a Sra. Dursley, da rua dos Alfeneiros, no 4, se orgulhavam de dizer que eram perfeitamente normais, muito bem, obrigado. Eram as últimas pessoas no mundo que se esperaria que se metessem em alguma coisa estranha ou misteriosa, porque simplesmente não compactuavam com esse tipo de bobagem. O Sr. Dursley era diretor de uma firma chamada Grunnings, que fazia perfurações. Era um homem alto e corpulento quase sem pescoço, embora tivesse enormes bigodes. A Sra. Dursley era magra e loura e tinha um pescoço quase duas vezes mais comprido que o normal, o que era muito útil porque ela passava grande parte do tempo espichando-o por cima da cerca do jardim para espiar os vizinhos. Os Dursley tinham um filhinho chamado Dudley, o Duda, e em sua opinião não havia garoto melhor em nenhum lugar do mundo. Os Dursley tinham tudo que queriam, mas tinham também um segredo, e seu maior receio era que alguém o descobrisse. Achavam que não iriam aguentar se alguém descobrisse a existência dos Potter. A Sra. Potter era irmã da Sra. Dursley, mas não se viam havia muitos anos; na realidade a Sra. Dursley fingia que não tinha irmã, porque esta e o marido imprestável eram o que havia de menos parecido possível com os Dursley. Eles estremeciam só de pensar o que os vizinhos iriam dizer se os Potter aparecessem na rua. Os Dursley sabiam que os Potter tinham um filhinho, também, mas nunca o tinham visto. O garoto era mais uma razão para manter os Potter a distância; eles não queriam que Duda se misturasse com uma criança daquelas. Quando o Sr. e a Sra. Dursley acordaram na terça-feira monótona e cinzenta em que a nossa história começa, não havia nada no céu nublado lá fora sugerindo as coisas estranhas e misteriosas que não tardariam a acontecer por todo o país. O Sr. Dursley cantarolava ao escolher a gravata mais sem graça do mundo para ir trabalhar e a Sra. Dursley fofocava alegremente enquanto lutava para encaixar um Duda aos berros na cadeirinha alta. Nenhum deles reparou em uma coruja parda que passou, batendo as asas, pela janela. Às oito e meia, o Sr. Dursley apanhou a pasta, deu um beijinho no rosto da Sra. Dursley e tentou dar um beijo de despedida em Duda mas não conseguiu, porque na hora Duda estava tendo um acesso de raiva e atirava o cereal nas paredes. - Pestinha - disse rindo contrafeito o Sr. Dursley ao sair de casa. Entrou no carro e deu marcha à ré para sair do estacionamento do número quatro. Foi na esquina da rua que ele notou o primeiro indício de que algo estranho ocorria - um gato lia um mapa. Por um instante o Sr. Dursley não percebeu o que vira - em seguida virou rapidamente a cabeça para dar uma segunda olhada. Havia um gato de listras amarelas sentado na esquina da rua dos Alfeneiros, mas não havia nenhum mapa à vista. Em que estaria pensando naquela hora? Devia ter sido um efeito da luz. Ele piscou e arregalou os olhos para o gato. O gato o encarou. Enquanto virava a esquina e subia a rua, espiou o gato pelo espelho retrovisor. Ele agora estava lendo a placa que dizia rua dos Alfeneiros - não, estava olhando a placa: gatos não podiam ler mapas nem placas. O Sr. Dursley sacudiu a cabeça e tirou o gato do pensamento. Dur'),\n",
              " (1,\n",
              "  'ante o caminho para a cidade ele não pensou em mais nada exceto no grande pedido de brocas que tinha esperanças de receber naquele dia. Mas ao sair da cidade, as brocas foram varridas de sua cabeça por outra coisa. Ao parar no costumeiro engarrafamento matinal, não pôde deixar de notar que havia uma quantidade de gente estranhamente vestida andando pelas ruas. Gente com capas largas. O Sr. Dursley não tolerava gente que andava com roupas ridículas - os trapos que se viam nos jovens! Imaginou que aquilo fosse uma nova moda idiota. Tamborilou os dedos no volante e seu olhar recaiu em um grupinho de excêntricos parados bem perto dele. Cochichavam excitados. O Sr. Dursley se irritou ao ver que alguns deles nem eram jovens; ora, aquele homem devia ser mais velho do que ele, e usava uma capa verde-esmeralda! Que petulância! Mas então ocorreu ao Sr. Dursley que se tratava provavelmente de alguma promoção boba - essas pessoas estavam obviamente arrecadando alguma coisa... é, devia ser isto! O tráfego avançou e alguns minutos depois o Sr. Dursley chegou ao estacionamento da Grunnings, o pensamento de volta às brocas. O Sr. Dursley sempre sentava de costas para a janela em seu escritório no nono andar. Se não o fizesse, talvez tivesse achado mais difícil se concentrar em brocas aquela manhã. Ele não viu as corujas que voavam velozes em plena luz do dia, embora as pessoas na rua as vissem; elas apontavam e se espantavam enquanto coruja atrás de coruja passava no alto. A maioria jamais vira uma coruja mesmo à noite. O Sr. Dursley, porém, teve uma manhã perfeitamente normal sem corujas. Gritou com cinco pessoas diferentes. Deu vários telefonemas importantes e gritou mais um pouco. Estava de excelente humor até a hora do almoço, quando pensou em esticar as pernas e atravessar a rua para comprar um pãozinho doce na padaria defronte. Esquecera completamente as pessoas de capas até passar por um grupo delas próximo à padaria. Olhou-as com raiva ao passar. Não sabia o porquê, mas elas o deixavam nervoso. Essas cochichavam agitadas, também, mas ele não viu nenhuma latinha de coleta. Foi ao passar por elas, na volta, levando uma grande rosca açucarada em um saco, que entreouviu algumas palavras do que diziam. - ... Os Potter, é verdade, foi o que ouvi... - ... é, o filho deles, Harry... O Sr. Dursley parou de repente. O medo invadiu-o. Virou a cabeça para olhar as pessoas que cochichavam como se quisesse dizer alguma coisa, mas pensou melhor. Atravessou a rua depressa, correu para o escritório, disse rispidamente à secretária que não o incomodasse, agarrou o telefone e quase terminara de discar o número de casa quando mudou de ideia. Pôs o fone no gancho e alisou os bigodes, pensando... não, estava agindo como um idiota. Potter não era um nome tão fora do comum assim. Tinha certeza de que havia muita gente chamada Potter com um filho chamado Harry. Pensando bem, nem sequer tinha certeza de que o sobrinho tivesse o nome de Harry. Jamais vira o menino. Talvez fosse Ernesto. Ou Eduardo. Não tinha sentido preocupar a Sra. Dursley, ela sempre ficava tão perturbada à simples menção da irmã. Não a culpava - se ele tivesse uma irmã como aquela... mas, mesmo assim, aquelas pessoas de capas... Achou bem mais difícil se concentrar nas brocas aquela tarde e, quando deixou o edifício às cinco horas, continuava tão preocupado que deu um encontrão'),\n",
              " (2,\n",
              "  'em alguém parado ali à porta. - Desculpe - murmurou, quando o velhinho cambaleou e quase caiu. Levou alguns segundos até o Sr. Dursley perceber que o homem estava usando uma capa roxa. Não parecia nada aborrecido por ter sido quase jogado ao chão. Ao contrário, seu rosto se abriu em um largo sorriso e ele disse numa voz esganiçada que fez os passantes olharem: - Não precisa pedir desculpas, caro senhor, porque nada poderia me aborrecer hoje! Alegre-se, porque Você-Sabe-Quem finalmente foi-se embora! Até trouxas como o senhor deviam estar comemorando um dia tão feliz! E o velho abraçou o Sr. Dursley pela cintura e se afastou. O Sr. Dursley ficou pregado no chão. Fora abraçado por um completo estranho. E também achava que fora chamado de trouxa, o que quer que isso quisesse dizer. Estava abalado. Correu para o carro e partiu para casa, esperando que estivesse imaginando coisas, o que nunca esperara que fizesse, porque não aprovava a imaginação. Quando entrou no estacionamento do número quatro, a primeira coisa que viu - e isso não melhorou o seu estado de espírito - foi o gato listrado que notara aquela manhã. Agora ele estava sentado no muro do jardim. Tinha certeza de que era o mesmo; as marcas em volta dos olhos eram as mesmas. - Chispa! - disse o Sr. Dursley em voz alta. O gato não se mexeu. Apenas lançou-lhe um olhar severo. Será que isto era um comportamento normal para um gato?, pensou o Sr. Dursley. Continuava decidido a não comentar nada com a esposa. A Sra. Dursley tivera um dia normal e agradável. Contou-lhe durante o jantar os problemas da senhora do lado com a filha e ainda que Duda aprendera uma palavra nova (\"Nunca\"). O Sr. Dursley tentou agir normalmente. Depois que Duda foi se deitar, ele chegou à sala em tempo de ouvir o último noticiário noturno. \"E, por último, os observadores de pássaros em toda parte registraram que as corujas do país se comportaram de forma muito estranha hoje. Embora elas normalmente cacem à noite e raramente apareçam à luz do dia, centenas desses pássaros foram vistos hoje voando em todas as direções desde o alvorecer. Os especialistas não sabem explicar por que as corujas de repente mudaram o seu padrão de sono.\" O locutor se permitiu um sorriso. \"Muito misterioso. E agora, com Jorge Mendes, o nosso boletim meteorológico. Vai haver mais tempestades de corujas hoje à noite, Jorge?\" \"Bom, Eduardo\", disse o meteorologista, \"não sei lhe dizer, mas não foram só as corujas que se comportaram de modo estranho hoje. Ouvintes de todo o país têm telefonado para reclamar que em vez do aguaceiro que prometi para ontem, eles têm tido chuvas de estrelas! Talvez alguém ande festejando a noite das fogueiras uma semana mais cedo este ano! Mas posso prometer para hoje uma noite chuvosa.\" O Sr. Dursley ficou paralisado na poltrona. Estrelas cadentes em todo o país? Corujas voando durante o dia? Gente misteriosa usando capas por todo lado? E um cochicho, um cochicho a respeito dos Potter... A Sra. Dursley entrou na sala trazendo duas xícaras de chá. Não adiantava. Teria que lhe dizer alguma coisa. Pigarreou nervoso. - Hum, hum, Petúnia, querida, você não tem tido notícias de sua irmã, ultimamente? Conforme esperava, a Sra. Dursley pareceu chocada e ab')]"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "PpOMDrekTIzB",
        "outputId": "246a13dd-befb-48d9-83c5-ac400a88bceb"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "     query_id  0  doc_id  rel\n",
              "0           0  0       0    1\n",
              "1           1  0       1    1\n",
              "2           2  0       2    1\n",
              "3           3  0       3    1\n",
              "4           4  0       4    1\n",
              "..        ... ..     ...  ...\n",
              "135       135  0     135    1\n",
              "136       136  0     136    1\n",
              "137       137  0     137    1\n",
              "138       138  0     138    1\n",
              "139       139  0     139    1\n",
              "\n",
              "[140 rows x 4 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-797c6ad5-3368-42c1-82bf-05f29e7c9fc4\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>query_id</th>\n",
              "      <th>0</th>\n",
              "      <th>doc_id</th>\n",
              "      <th>rel</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>135</th>\n",
              "      <td>135</td>\n",
              "      <td>0</td>\n",
              "      <td>135</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>136</th>\n",
              "      <td>136</td>\n",
              "      <td>0</td>\n",
              "      <td>136</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>137</th>\n",
              "      <td>137</td>\n",
              "      <td>0</td>\n",
              "      <td>137</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>138</th>\n",
              "      <td>138</td>\n",
              "      <td>0</td>\n",
              "      <td>138</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>139</th>\n",
              "      <td>139</td>\n",
              "      <td>0</td>\n",
              "      <td>139</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>140 rows × 4 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-797c6ad5-3368-42c1-82bf-05f29e7c9fc4')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-797c6ad5-3368-42c1-82bf-05f29e7c9fc4 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-797c6ad5-3368-42c1-82bf-05f29e7c9fc4');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-35f2aa3d-c92f-4c0c-bd05-a329ccaebb3e\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-35f2aa3d-c92f-4c0c-bd05-a329ccaebb3e')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-35f2aa3d-c92f-4c0c-bd05-a329ccaebb3e button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "  <div id=\"id_2a9ebc04-9f4b-434c-9720-4230f3071773\">\n",
              "    <style>\n",
              "      .colab-df-generate {\n",
              "        background-color: #E8F0FE;\n",
              "        border: none;\n",
              "        border-radius: 50%;\n",
              "        cursor: pointer;\n",
              "        display: none;\n",
              "        fill: #1967D2;\n",
              "        height: 32px;\n",
              "        padding: 0 0 0 0;\n",
              "        width: 32px;\n",
              "      }\n",
              "\n",
              "      .colab-df-generate:hover {\n",
              "        background-color: #E2EBFA;\n",
              "        box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "        fill: #174EA6;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate {\n",
              "        background-color: #3B4455;\n",
              "        fill: #D2E3FC;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate:hover {\n",
              "        background-color: #434B5C;\n",
              "        box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "        filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "        fill: #FFFFFF;\n",
              "      }\n",
              "    </style>\n",
              "    <button class=\"colab-df-generate\" onclick=\"generateWithVariable('qrels')\"\n",
              "            title=\"Generate code using this dataframe.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M7,19H8.4L18.45,9,17,7.55,7,17.6ZM5,21V16.75L18.45,3.32a2,2,0,0,1,2.83,0l1.4,1.43a1.91,1.91,0,0,1,.58,1.4,1.91,1.91,0,0,1-.58,1.4L9.25,21ZM18.45,9,17,7.55Zm-12,3A5.31,5.31,0,0,0,4.9,8.1,5.31,5.31,0,0,0,1,6.5,5.31,5.31,0,0,0,4.9,4.9,5.31,5.31,0,0,0,6.5,1,5.31,5.31,0,0,0,8.1,4.9,5.31,5.31,0,0,0,12,6.5,5.46,5.46,0,0,0,6.5,12Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "    <script>\n",
              "      (() => {\n",
              "      const buttonEl =\n",
              "        document.querySelector('#id_2a9ebc04-9f4b-434c-9720-4230f3071773 button.colab-df-generate');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      buttonEl.onclick = () => {\n",
              "        google.colab.notebook.generateWithVariable('qrels');\n",
              "      }\n",
              "      })();\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ],
      "source": [
        "# load qrels FAQ like\n",
        "qrels = pd.read_csv(\"/content/qrels.tsv\", sep = \"\\t\", header = None)\n",
        "qrels.columns = [\"query_id\", \"0\", \"doc_id\", \"rel\"]\n",
        "qrels"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def num_tokens_from_string(string: str, encoding_name: str) -> int:\n",
        "    \"\"\"Returns the number of tokens in a text string.\"\"\"\n",
        "    encoding = tiktoken.encoding_for_model(\"text-embedding-ada-002\")\n",
        "    num_tokens = len(encoding.encode(string))\n",
        "    return num_tokens"
      ],
      "metadata": {
        "id": "u-IWdGoWXkgx"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "doc_lengths = pd.DataFrame(corpus.items(), columns = [\"id\", \"text\"])\n",
        "doc_lengths[\"n_tokens\"] = [num_tokens_from_string(corpus[idx], \"cl100k_base\") for idx in corpus]\n",
        "doc_lengths[\"n_tokens\"].describe()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PKxG5e9HeZFq",
        "outputId": "74368f1e-b82e-4753-ac95-3f847827fea5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "count     140.000000\n",
              "mean      992.235714\n",
              "std        73.317837\n",
              "min       131.000000\n",
              "25%       998.000000\n",
              "50%       999.000000\n",
              "75%       999.000000\n",
              "max      1001.000000\n",
              "Name: n_tokens, dtype: float64"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Build Folders"
      ],
      "metadata": {
        "id": "ByBH5PTPJrB4"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!rm -rf runs\n",
        "!rm -rf evals\n",
        "!mkdir runs\n",
        "!mkdir evals"
      ],
      "metadata": {
        "id": "g0ZePSJSJtGU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "trec_results = {}"
      ],
      "metadata": {
        "id": "PBLQVkmbKqFG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NRp0VIA3bquq"
      },
      "source": [
        "## BM25\n",
        "\n",
        "* https://github.com/castorini/anserini/blob/master/src/main/resources/regression/msmarco-passage.yaml\n",
        "* https://github.com/castorini/pyserini/blob/f75adca8c410e64b3ff1375e181a0ea3af1ddb28/docs/usage-index.md"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "B_2Xm8xgbwl8",
        "outputId": "d41d4d7c-2021-45d5-b627-3d2884192551"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Writing file on /content/collections/corpus.jsonl: 100%|██████████| 140/140 [00:00<00:00, 14854.61it/s]\n"
          ]
        }
      ],
      "source": [
        "!rm -rf /content/collections\n",
        "!mkdir /content/collections\n",
        "\n",
        "# Save corpus as jsonl file\n",
        "output_json = f'/content/collections/corpus.jsonl'\n",
        "\n",
        "with open(output_json,'w') as f_out:\n",
        "    for doc_id, doc_text in tqdm(corpus.items(), desc=f'Writing file on {output_json}'):\n",
        "        output_dict = {'id': doc_id, 'contents': doc_text}\n",
        "        f_out.write(json.dumps(output_dict, ensure_ascii=False) + '\\n')"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!rm -rf indexes\n",
        "\n",
        "# Create JsonCollection from jsonl file\n",
        "!python -m pyserini.index.lucene \\\n",
        "--collection JsonCollection \\\n",
        "--input /content/collections \\\n",
        "--index indexes/lucene-index-trec-hp \\\n",
        "--generator DefaultLuceneDocumentGenerator \\\n",
        "--threads 9 \\\n",
        "--language pt \\\n",
        "--storePositions \\\n",
        "--storeDocvectors \\\n",
        "--storeRaw"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "X84Rn-6qucsy",
        "outputId": "21d23689-f888-468d-b060-aef195a007d0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "WARNING: sun.reflect.Reflection.getCallerClass is not supported. This will impact performance.\n",
            "2024-01-09 03:58:20,904 INFO  [main] index.AbstractIndexer (AbstractIndexer.java:204) - Setting log level to INFO\n",
            "2024-01-09 03:58:20,907 INFO  [main] index.AbstractIndexer (AbstractIndexer.java:207) - ============ Loading Index Configuration ============\n",
            "2024-01-09 03:58:20,907 INFO  [main] index.AbstractIndexer (AbstractIndexer.java:208) - AbstractIndexer settings:\n",
            "2024-01-09 03:58:20,907 INFO  [main] index.AbstractIndexer (AbstractIndexer.java:209) -  + DocumentCollection path: /content/collections\n",
            "2024-01-09 03:58:20,908 INFO  [main] index.AbstractIndexer (AbstractIndexer.java:210) -  + CollectionClass: JsonCollection\n",
            "2024-01-09 03:58:20,908 INFO  [main] index.AbstractIndexer (AbstractIndexer.java:211) -  + Index path: indexes/lucene-index-trec-hp\n",
            "2024-01-09 03:58:20,908 INFO  [main] index.AbstractIndexer (AbstractIndexer.java:212) -  + Threads: 9\n",
            "2024-01-09 03:58:20,909 INFO  [main] index.AbstractIndexer (AbstractIndexer.java:213) -  + Optimize (merge segments)? false\n",
            "2024-01-09 03:58:20,945 INFO  [main] index.IndexCollection (IndexCollection.java:237) - Using language-specific analyzer\n",
            "2024-01-09 03:58:20,946 INFO  [main] index.IndexCollection (IndexCollection.java:238) - Language: pt\n",
            "2024-01-09 03:58:21,098 INFO  [main] index.IndexCollection (IndexCollection.java:197) - IndexCollection settings:\n",
            "2024-01-09 03:58:21,099 INFO  [main] index.IndexCollection (IndexCollection.java:198) -  + Generator: DefaultLuceneDocumentGenerator\n",
            "2024-01-09 03:58:21,099 INFO  [main] index.IndexCollection (IndexCollection.java:199) -  + Language: pt\n",
            "2024-01-09 03:58:21,099 INFO  [main] index.IndexCollection (IndexCollection.java:200) -  + Stemmer: porter\n",
            "2024-01-09 03:58:21,100 INFO  [main] index.IndexCollection (IndexCollection.java:201) -  + Keep stopwords? false\n",
            "2024-01-09 03:58:21,100 INFO  [main] index.IndexCollection (IndexCollection.java:202) -  + Stopwords: null\n",
            "2024-01-09 03:58:21,100 INFO  [main] index.IndexCollection (IndexCollection.java:203) -  + Store positions? true\n",
            "2024-01-09 03:58:21,100 INFO  [main] index.IndexCollection (IndexCollection.java:204) -  + Store docvectors? true\n",
            "2024-01-09 03:58:21,101 INFO  [main] index.IndexCollection (IndexCollection.java:205) -  + Store document \"contents\" field? false\n",
            "2024-01-09 03:58:21,101 INFO  [main] index.IndexCollection (IndexCollection.java:206) -  + Store document \"raw\" field? true\n",
            "2024-01-09 03:58:21,101 INFO  [main] index.IndexCollection (IndexCollection.java:207) -  + Additional fields to index: []\n",
            "2024-01-09 03:58:21,101 INFO  [main] index.IndexCollection (IndexCollection.java:208) -  + Whitelist: null\n",
            "2024-01-09 03:58:21,102 INFO  [main] index.IndexCollection (IndexCollection.java:209) -  + Pretokenized?: false\n",
            "2024-01-09 03:58:21,102 INFO  [main] index.IndexCollection (IndexCollection.java:210) -  + Codec: Lucene99\n",
            "2024-01-09 03:58:21,102 INFO  [main] index.AbstractIndexer (AbstractIndexer.java:237) - ============ Indexing Collection ============\n",
            "2024-01-09 03:58:21,104 INFO  [main] index.AbstractIndexer (AbstractIndexer.java:246) - Thread pool with 9 threads initialized.\n",
            "2024-01-09 03:58:21,105 INFO  [main] index.AbstractIndexer (AbstractIndexer.java:247) - 1 file found in /content/collections \n",
            "2024-01-09 03:58:21,105 INFO  [main] index.AbstractIndexer (AbstractIndexer.java:248) - Starting to index...\n",
            "2024-01-09 03:58:21,617 DEBUG [pool-2-thread-1] index.AbstractIndexer$IndexerThread (AbstractIndexer.java:175) - collections/corpus.jsonl: 140 docs added.\n",
            "2024-01-09 03:58:21,912 INFO  [main] index.AbstractIndexer (AbstractIndexer.java:306) - Indexing Complete! 140 documents indexed\n",
            "2024-01-09 03:58:21,913 INFO  [main] index.AbstractIndexer (AbstractIndexer.java:307) - ============ Final Counter Values ============\n",
            "2024-01-09 03:58:21,913 INFO  [main] index.AbstractIndexer (AbstractIndexer.java:308) - indexed:              140\n",
            "2024-01-09 03:58:21,913 INFO  [main] index.AbstractIndexer (AbstractIndexer.java:309) - unindexable:            0\n",
            "2024-01-09 03:58:21,913 INFO  [main] index.AbstractIndexer (AbstractIndexer.java:310) - empty:                  0\n",
            "2024-01-09 03:58:21,914 INFO  [main] index.AbstractIndexer (AbstractIndexer.java:311) - skipped:                0\n",
            "2024-01-09 03:58:21,915 INFO  [main] index.AbstractIndexer (AbstractIndexer.java:312) - errors:                 0\n",
            "2024-01-09 03:58:21,921 INFO  [main] index.AbstractIndexer (AbstractIndexer.java:315) - Total 140 documents indexed in 00:00:00\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7JIrSy0xb-M_",
        "outputId": "25085b21-1e0a-4e4c-f0e5-86de674bb7be"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Setting BM25 parameters: k1=0.82, b=0.68\n",
            "Running topics.tsv topics, saving to runs/run.hp.bm25.lucene.txt...\n",
            "100% 140/140 [00:02<00:00, 69.08it/s]\n"
          ]
        }
      ],
      "source": [
        "# Utiliza o pacote pyserini para ranquear\n",
        "experiment = \"bm25.lucene\"\n",
        "run_file = f\"runs/run.hp.{experiment}.txt\"\n",
        "eval_file = f\"evals/eval.hp.{experiment}.txt\"\n",
        "\n",
        "!rm -rf $run_file\n",
        "!rm -rf $eval_file\n",
        "\n",
        "!python -m pyserini.search.lucene \\\n",
        "  --index \"indexes/lucene-index-trec-hp\" \\\n",
        "  --topics \"topics.tsv\" \\\n",
        "  --output $run_file \\\n",
        "  --output-format trec \\\n",
        "  --language pt \\\n",
        "  --hits 1000 \\\n",
        "  --bm25 --k1 0.82 --b 0.68\n",
        "\n",
        "!/content/pyserini/tools/eval/trec_eval.9.0.4/trec_eval -c -M 10 -m recip_rank -m recall.3,5,7,9 qrels.tsv $run_file > $eval_file 2>&1\n",
        "trec_aux = pd.read_csv(eval_file, sep = \"\\t\", names = [\"metric\", \"all\", \"value\"])\n",
        "trec_aux[\"experiment\"] = experiment\n",
        "trec_results[experiment] = trec_aux.copy()"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "trec_results[experiment]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "BrcQ2eVcibAY",
        "outputId": "f0b0088f-9700-4af4-de81-300beeb8264b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                   metric  all   value   experiment\n",
              "0  recip_rank              all  0.8794  bm25.lucene\n",
              "1  recall_3                all  0.9143  bm25.lucene\n",
              "2  recall_5                all  0.9714  bm25.lucene\n",
              "3  recall_7                all  0.9857  bm25.lucene\n",
              "4  recall_9                all  0.9857  bm25.lucene"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-f7cec999-a4c9-452b-b8bc-6259edbabf57\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>metric</th>\n",
              "      <th>all</th>\n",
              "      <th>value</th>\n",
              "      <th>experiment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>recip_rank</td>\n",
              "      <td>all</td>\n",
              "      <td>0.8794</td>\n",
              "      <td>bm25.lucene</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>recall_3</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9143</td>\n",
              "      <td>bm25.lucene</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>recall_5</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9714</td>\n",
              "      <td>bm25.lucene</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>recall_7</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9857</td>\n",
              "      <td>bm25.lucene</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>recall_9</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9857</td>\n",
              "      <td>bm25.lucene</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-f7cec999-a4c9-452b-b8bc-6259edbabf57')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-f7cec999-a4c9-452b-b8bc-6259edbabf57 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-f7cec999-a4c9-452b-b8bc-6259edbabf57');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-3effd811-89d7-4e0d-89a5-3b0cfc8621d6\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-3effd811-89d7-4e0d-89a5-3b0cfc8621d6')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-3effd811-89d7-4e0d-89a5-3b0cfc8621d6 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dhetQiUEc2Rn"
      },
      "source": [
        "##  ADA002"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "aKMY_U2XffTF"
      },
      "outputs": [],
      "source": [
        "from google.colab import userdata\n",
        "from openai import OpenAI"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "RUkOjlazffVW"
      },
      "outputs": [],
      "source": [
        "client = OpenAI(api_key=userdata.get('openai-leonardo'))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8MP8t78odPrm"
      },
      "outputs": [],
      "source": [
        "def get_embedding(texts, model=\"text-embedding-ada-002\"):\n",
        "   return [emb.embedding for emb in client.embeddings.create(input = texts, model=model).data]"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def ada002_search(corpus, topics):\n",
        "\n",
        "    # ada002 search\n",
        "    df = pd.DataFrame(list(corpus.items()), columns=['id', 'conteudo'])\n",
        "\n",
        "    df[\"pergunta\"] = df[\"id\"].map(topics)\n",
        "    df[\"pergunta_emb\"] = get_embedding(df[\"pergunta\"].values.tolist())\n",
        "    df[\"conteudo_emb\"] = get_embedding(df[\"conteudo\"].values.tolist())\n",
        "\n",
        "    query_matrix = np.vstack(df[\"pergunta_emb\"])\n",
        "    contents_matrix = np.vstack(df[\"conteudo_emb\"])\n",
        "    similarity_matrix = np.dot(query_matrix, contents_matrix.T)\n",
        "\n",
        "    ids = df[\"id\"].values.tolist()\n",
        "    similarity = {}\n",
        "    for query_id, hits in zip(ids, similarity_matrix):\n",
        "      if similarity.get(query_id) is None:\n",
        "          similarity[query_id] = []\n",
        "      for doc_id, hit in zip(ids, hits):\n",
        "          similarity[query_id].append((doc_id, hit))\n",
        "\n",
        "    sorted_similarity = {key: sorted(value, key=lambda x: x[1], reverse=True) for key, value in similarity.items()}\n",
        "\n",
        "    return sorted_similarity, df"
      ],
      "metadata": {
        "id": "uAhndEMiNGr2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sorted_similarity, df_ada002 = ada002_search(corpus, topics)\n",
        "\n",
        "# Save run file\n",
        "experiment = \"ada002\"\n",
        "run_file = f\"runs/run.hp.{experiment}.txt\"\n",
        "eval_file = f\"evals/eval.hp.{experiment}.txt\"\n",
        "\n",
        "!rm -rf $run_file\n",
        "!rm -rf $eval_file\n",
        "\n",
        "with open(run_file,'w') as f_out:\n",
        "    for query_id, hits in tqdm(sorted_similarity.items(), desc=f'Writing file on {run_file}'):\n",
        "        rank = 1\n",
        "        for doc_id, score in hits:\n",
        "            f_out.write(f'{query_id}\\tQ0\\t{doc_id}\\t{rank}\\t{score}\\t{\"ADA002\"}\\n')\n",
        "            rank+=1\n",
        "\n",
        "!/content/pyserini/tools/eval/trec_eval.9.0.4/trec_eval -c -M 10 -m recip_rank -m recall.3,5,7,9 qrels.tsv $run_file > $eval_file 2>&1\n",
        "trec_aux = pd.read_csv(eval_file, sep = \"\\t\", names = [\"metric\", \"all\", \"value\"])\n",
        "trec_aux[\"experiment\"] = experiment\n",
        "trec_results[experiment] = trec_aux.copy()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4JeheNTaotBL",
        "outputId": "4c7b00b7-583a-46b4-fe99-bf49eda494ec"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Writing file on runs/run.hp.ada002.txt: 100%|██████████| 140/140 [00:00<00:00, 4481.13it/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_ada002.to_csv(\"df_ada002.csv\", sep = \"\\t\", index = False)"
      ],
      "metadata": {
        "id": "XnNag5tGiv1x"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "trec_results[experiment]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "7gr_iQ3iigLv",
        "outputId": "b646aef6-2e4b-478a-c9be-9b28a45dbe2e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                   metric  all   value experiment\n",
              "0  recip_rank              all  0.5839     ada002\n",
              "1  recall_3                all  0.6214     ada002\n",
              "2  recall_5                all  0.7286     ada002\n",
              "3  recall_7                all  0.8000     ada002\n",
              "4  recall_9                all  0.8714     ada002"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-907555cf-910c-486b-a745-a5afda54b7e3\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>metric</th>\n",
              "      <th>all</th>\n",
              "      <th>value</th>\n",
              "      <th>experiment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>recip_rank</td>\n",
              "      <td>all</td>\n",
              "      <td>0.5839</td>\n",
              "      <td>ada002</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>recall_3</td>\n",
              "      <td>all</td>\n",
              "      <td>0.6214</td>\n",
              "      <td>ada002</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>recall_5</td>\n",
              "      <td>all</td>\n",
              "      <td>0.7286</td>\n",
              "      <td>ada002</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>recall_7</td>\n",
              "      <td>all</td>\n",
              "      <td>0.8000</td>\n",
              "      <td>ada002</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>recall_9</td>\n",
              "      <td>all</td>\n",
              "      <td>0.8714</td>\n",
              "      <td>ada002</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-907555cf-910c-486b-a745-a5afda54b7e3')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-907555cf-910c-486b-a745-a5afda54b7e3 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-907555cf-910c-486b-a745-a5afda54b7e3');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-0645a24b-4047-4295-a151-9fe7e2b243d9\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-0645a24b-4047-4295-a151-9fe7e2b243d9')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-0645a24b-4047-4295-a151-9fe7e2b243d9 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Hybrid Search (BM25 + ADA002)\n",
        "\n",
        "* https://plg.uwaterloo.ca/~gvcormac/cormacksigir09-rrf.pdf"
      ],
      "metadata": {
        "id": "0BT0FQykqfBO"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# save run file\n",
        "rrf_k_list = [0, 60]\n",
        "for rrf_k in rrf_k_list:\n",
        "    experiment = f\"hybrid_bm25_ada002_k_{rrf_k}\"\n",
        "    run_file = f'runs/run.hp.{experiment}.txt'\n",
        "    eval_file = f'evals/eval.hp.{experiment}.txt'\n",
        "\n",
        "    run_file1_path = 'runs/run.hp.bm25.lucene.txt'\n",
        "    run_file2_path = 'runs/run.hp.ada002.txt'\n",
        "\n",
        "    !python -m pyserini.fusion \\\n",
        "      --runs $run_file1_path $run_file2_path \\\n",
        "      --output $run_file \\\n",
        "      --rrf.k $rrf_k \\\n",
        "      --k 1000\n",
        "\n",
        "    !/content/pyserini/tools/eval/trec_eval.9.0.4/trec_eval -c -M 10 -m recip_rank -m recall.3,5,7,9 qrels.tsv $run_file > $eval_file 2>&1\n",
        "    trec_aux = pd.read_csv(eval_file, sep = \"\\t\", names = [\"metric\", \"all\", \"value\"])\n",
        "    trec_aux[\"experiment\"] = experiment\n",
        "    trec_results[experiment] = trec_aux.copy()"
      ],
      "metadata": {
        "id": "umzUkommzLb-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "rrf_k_list = [0, 60]\n",
        "for rrf_k in rrf_k_list:\n",
        "    experiment = f\"hybrid_bm25_ada002_k_{rrf_k}\"\n",
        "    display(trec_results[experiment])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 396
        },
        "id": "ah698XsVIqF5",
        "outputId": "d94b4e86-fcb0-4945-f4ed-6ed9f5951fca"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "                   metric  all   value              experiment\n",
              "0  recip_rank              all  0.8177  hybrid_bm25_ada002_k_0\n",
              "1  recall_3                all  0.9429  hybrid_bm25_ada002_k_0\n",
              "2  recall_5                all  0.9786  hybrid_bm25_ada002_k_0\n",
              "3  recall_7                all  0.9857  hybrid_bm25_ada002_k_0\n",
              "4  recall_9                all  0.9929  hybrid_bm25_ada002_k_0"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-18470932-0c80-4397-85f7-919a25538012\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>metric</th>\n",
              "      <th>all</th>\n",
              "      <th>value</th>\n",
              "      <th>experiment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>recip_rank</td>\n",
              "      <td>all</td>\n",
              "      <td>0.8177</td>\n",
              "      <td>hybrid_bm25_ada002_k_0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>recall_3</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9429</td>\n",
              "      <td>hybrid_bm25_ada002_k_0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>recall_5</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9786</td>\n",
              "      <td>hybrid_bm25_ada002_k_0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>recall_7</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9857</td>\n",
              "      <td>hybrid_bm25_ada002_k_0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>recall_9</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9929</td>\n",
              "      <td>hybrid_bm25_ada002_k_0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-18470932-0c80-4397-85f7-919a25538012')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-18470932-0c80-4397-85f7-919a25538012 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-18470932-0c80-4397-85f7-919a25538012');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-2a95e717-2070-47d3-ac25-f8c35bb3ee56\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-2a95e717-2070-47d3-ac25-f8c35bb3ee56')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-2a95e717-2070-47d3-ac25-f8c35bb3ee56 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "                   metric  all   value               experiment\n",
              "0  recip_rank              all  0.7702  hybrid_bm25_ada002_k_60\n",
              "1  recall_3                all  0.8714  hybrid_bm25_ada002_k_60\n",
              "2  recall_5                all  0.9143  hybrid_bm25_ada002_k_60\n",
              "3  recall_7                all  0.9286  hybrid_bm25_ada002_k_60\n",
              "4  recall_9                all  0.9500  hybrid_bm25_ada002_k_60"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-733e2091-c6b2-4b5a-b76b-0b2b1a0f73a4\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>metric</th>\n",
              "      <th>all</th>\n",
              "      <th>value</th>\n",
              "      <th>experiment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>recip_rank</td>\n",
              "      <td>all</td>\n",
              "      <td>0.7702</td>\n",
              "      <td>hybrid_bm25_ada002_k_60</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>recall_3</td>\n",
              "      <td>all</td>\n",
              "      <td>0.8714</td>\n",
              "      <td>hybrid_bm25_ada002_k_60</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>recall_5</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9143</td>\n",
              "      <td>hybrid_bm25_ada002_k_60</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>recall_7</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9286</td>\n",
              "      <td>hybrid_bm25_ada002_k_60</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>recall_9</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9500</td>\n",
              "      <td>hybrid_bm25_ada002_k_60</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-733e2091-c6b2-4b5a-b76b-0b2b1a0f73a4')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-733e2091-c6b2-4b5a-b76b-0b2b1a0f73a4 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-733e2091-c6b2-4b5a-b76b-0b2b1a0f73a4');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-7cdab8fe-f6c5-42c6-81e8-de01fc85a1eb\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-7cdab8fe-f6c5-42c6-81e8-de01fc85a1eb')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-7cdab8fe-f6c5-42c6-81e8-de01fc85a1eb button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## ADA002 CUSTOM"
      ],
      "metadata": {
        "id": "LVFrlYg__e4G"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def embedding_multiplied_by_matrix(embedding, matrix):\n",
        "    embedding_tensor = torch.tensor(embedding).float()\n",
        "    modified_embedding = torch.einsum(\"b, bc -> c\", embedding_tensor, matrix)\n",
        "\n",
        "    modified_embedding = modified_embedding.detach().numpy()\n",
        "\n",
        "    return modified_embedding\n",
        "\n",
        "\n",
        "def apply_matrix_to_embeddings_dataframe(matrix, df):\n",
        "    def cosine_similarity(a, b):\n",
        "        return np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b))\n",
        "\n",
        "\n",
        "    for column in [\"pergunta_embedding\", \"chunk_embedding\"]:\n",
        "        df[f\"{column}_custom\"] = df[column].apply(\n",
        "            lambda x: embedding_multiplied_by_matrix(x, matrix)\n",
        "        )\n",
        "    df[\"cosine_similarity_custom\"] = df.apply(\n",
        "        lambda row: cosine_similarity(\n",
        "            row[\"pergunta_embedding_custom\"], row[\"chunk_embedding_custom\"]\n",
        "        ),\n",
        "        axis=1,\n",
        "    )"
      ],
      "metadata": {
        "id": "kinQh85RB1Fv"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df_custom = pd.read_csv(f\"{path}/tozip/df_final_to_customize_embds_HP_2negs.csv\")"
      ],
      "metadata": {
        "id": "eJOKV_Qb_now"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df_custom.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 293
        },
        "id": "iHNcQdekCKwR",
        "outputId": "4dc0f11a-31ad-493b-c240-6cb005db4f05"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                            pergunta  \\\n",
              "0  Qual é o segredo que o Sr. e a Sra. Dursley te...   \n",
              "1  Qual era a cor da capa que o Sr. Dursley viu u...   \n",
              "2  Qual foi a reação do Sr. Dursley ao ser abraça...   \n",
              "3  Qual é o nome completo do personagem que apare...   \n",
              "4  Qual objeto Dumbledore utilizou para apagar os...   \n",
              "\n",
              "                                            contexto  \\\n",
              "0  Os Dursley tinham tudo que queriam, mas tinham...   \n",
              "1  O Sr. Dursley não tolerava gente que andava co...   \n",
              "2  Levou alguns segundos até o Sr. Dursley perceb...   \n",
              "3  Ninguém jamais vislumbrara nada parecido com e...   \n",
              "4  Encontrou o que procurava no bolso interior da...   \n",
              "\n",
              "                                            resposta  \\\n",
              "0                            a existência dos Potter   \n",
              "1                           uma capa verde-esmeralda   \n",
              "2  O Sr. Dursley ficou pregado no chão. Fora abra...   \n",
              "3                   O nome dele era Alvo Dumbledore.   \n",
              "4              Ele acionou o 'apagueiro' doze vezes.   \n",
              "\n",
              "                                               chunk  label dataset  \\\n",
              "0  - CAPÍTULO UM - O menino que sobreviveu O Sr. ...      1   train   \n",
              "1  ante o caminho para a cidade ele não pensou em...      1   train   \n",
              "2  em alguém parado ali à porta. - Desculpe - mur...      1   train   \n",
              "3  orrecida. Afinal, normalmente fingiam que ela ...      1   train   \n",
              "4  pando a capa, procurando alguma coisa. Mas par...      1   train   \n",
              "\n",
              "                                  pergunta_embedding  \\\n",
              "0  [0.022689364850521088, 0.0038326631765812635, ...   \n",
              "1  [0.005216388031840324, 0.0008732206770218909, ...   \n",
              "2  [-0.009695776738226414, -0.0014956077793613076...   \n",
              "3  [0.004215216264128685, 0.00600033113732934, 0....   \n",
              "4  [0.001769782043993473, 0.012089592404663563, 0...   \n",
              "\n",
              "                                     chunk_embedding  cosine_similarity  \n",
              "0  [0.014419169165194035, 0.011491919867694378, -...           0.873794  \n",
              "1  [-0.002306348877027631, -0.0014758666511625051...           0.873999  \n",
              "2  [0.004920588340610266, 0.004709234461188316, 0...           0.884103  \n",
              "3  [0.008049095049500465, -0.0045163994655013084,...           0.812161  \n",
              "4  [0.007251789793372154, 0.003910213243216276, 0...           0.879149  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-8656a68f-5e16-4f33-89d0-e8e08960ae86\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>pergunta</th>\n",
              "      <th>contexto</th>\n",
              "      <th>resposta</th>\n",
              "      <th>chunk</th>\n",
              "      <th>label</th>\n",
              "      <th>dataset</th>\n",
              "      <th>pergunta_embedding</th>\n",
              "      <th>chunk_embedding</th>\n",
              "      <th>cosine_similarity</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Qual é o segredo que o Sr. e a Sra. Dursley te...</td>\n",
              "      <td>Os Dursley tinham tudo que queriam, mas tinham...</td>\n",
              "      <td>a existência dos Potter</td>\n",
              "      <td>- CAPÍTULO UM - O menino que sobreviveu O Sr. ...</td>\n",
              "      <td>1</td>\n",
              "      <td>train</td>\n",
              "      <td>[0.022689364850521088, 0.0038326631765812635, ...</td>\n",
              "      <td>[0.014419169165194035, 0.011491919867694378, -...</td>\n",
              "      <td>0.873794</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Qual era a cor da capa que o Sr. Dursley viu u...</td>\n",
              "      <td>O Sr. Dursley não tolerava gente que andava co...</td>\n",
              "      <td>uma capa verde-esmeralda</td>\n",
              "      <td>ante o caminho para a cidade ele não pensou em...</td>\n",
              "      <td>1</td>\n",
              "      <td>train</td>\n",
              "      <td>[0.005216388031840324, 0.0008732206770218909, ...</td>\n",
              "      <td>[-0.002306348877027631, -0.0014758666511625051...</td>\n",
              "      <td>0.873999</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Qual foi a reação do Sr. Dursley ao ser abraça...</td>\n",
              "      <td>Levou alguns segundos até o Sr. Dursley perceb...</td>\n",
              "      <td>O Sr. Dursley ficou pregado no chão. Fora abra...</td>\n",
              "      <td>em alguém parado ali à porta. - Desculpe - mur...</td>\n",
              "      <td>1</td>\n",
              "      <td>train</td>\n",
              "      <td>[-0.009695776738226414, -0.0014956077793613076...</td>\n",
              "      <td>[0.004920588340610266, 0.004709234461188316, 0...</td>\n",
              "      <td>0.884103</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Qual é o nome completo do personagem que apare...</td>\n",
              "      <td>Ninguém jamais vislumbrara nada parecido com e...</td>\n",
              "      <td>O nome dele era Alvo Dumbledore.</td>\n",
              "      <td>orrecida. Afinal, normalmente fingiam que ela ...</td>\n",
              "      <td>1</td>\n",
              "      <td>train</td>\n",
              "      <td>[0.004215216264128685, 0.00600033113732934, 0....</td>\n",
              "      <td>[0.008049095049500465, -0.0045163994655013084,...</td>\n",
              "      <td>0.812161</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Qual objeto Dumbledore utilizou para apagar os...</td>\n",
              "      <td>Encontrou o que procurava no bolso interior da...</td>\n",
              "      <td>Ele acionou o 'apagueiro' doze vezes.</td>\n",
              "      <td>pando a capa, procurando alguma coisa. Mas par...</td>\n",
              "      <td>1</td>\n",
              "      <td>train</td>\n",
              "      <td>[0.001769782043993473, 0.012089592404663563, 0...</td>\n",
              "      <td>[0.007251789793372154, 0.003910213243216276, 0...</td>\n",
              "      <td>0.879149</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-8656a68f-5e16-4f33-89d0-e8e08960ae86')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-8656a68f-5e16-4f33-89d0-e8e08960ae86 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-8656a68f-5e16-4f33-89d0-e8e08960ae86');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-835e37da-1a93-49f7-b1b2-87ce54903b02\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-835e37da-1a93-49f7-b1b2-87ce54903b02')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-835e37da-1a93-49f7-b1b2-87ce54903b02 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_custom[\"pergunta_embedding\"] = df_custom[\"pergunta_embedding\"].apply(eval).apply(np.array)\n",
        "df_custom[\"chunk_embedding\"] = df_custom[\"chunk_embedding\"].apply(eval).apply(np.array)\n",
        "df_custom = df_custom.assign(cosine_similarity = df_custom.cosine_similarity.apply(lambda x: float(x)))"
      ],
      "metadata": {
        "id": "k7GoZPEnCJRy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#best_matrix = torch.load(f\"{path}best_matrix.zip\", map_location=\"cpu\")\n",
        "best_matrix = torch.load(f\"{path}best_matrix\", map_location=\"cpu\")\n",
        "apply_matrix_to_embeddings_dataframe(best_matrix, df_custom)\n",
        "df_custom.head(2)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 182
        },
        "id": "wspyggeM_hVs",
        "outputId": "f36cd7a0-6c0c-423f-dae2-d2242c0ec2cd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                            pergunta  \\\n",
              "0  Qual é o segredo que o Sr. e a Sra. Dursley te...   \n",
              "1  Qual era a cor da capa que o Sr. Dursley viu u...   \n",
              "\n",
              "                                            contexto  \\\n",
              "0  Os Dursley tinham tudo que queriam, mas tinham...   \n",
              "1  O Sr. Dursley não tolerava gente que andava co...   \n",
              "\n",
              "                   resposta  \\\n",
              "0   a existência dos Potter   \n",
              "1  uma capa verde-esmeralda   \n",
              "\n",
              "                                               chunk  label dataset  \\\n",
              "0  - CAPÍTULO UM - O menino que sobreviveu O Sr. ...      1   train   \n",
              "1  ante o caminho para a cidade ele não pensou em...      1   train   \n",
              "\n",
              "                                  pergunta_embedding  \\\n",
              "0  [0.022689364850521088, 0.0038326631765812635, ...   \n",
              "1  [0.005216388031840324, 0.0008732206770218909, ...   \n",
              "\n",
              "                                     chunk_embedding  cosine_similarity  \\\n",
              "0  [0.014419169165194035, 0.011491919867694378, -...           0.873794   \n",
              "1  [-0.002306348877027631, -0.0014758666511625051...           0.873999   \n",
              "\n",
              "                           pergunta_embedding_custom  \\\n",
              "0  [0.17453432, -0.60518175, -0.7326583, 0.358211...   \n",
              "1  [-0.62348986, -0.43539473, -0.32246643, -0.106...   \n",
              "\n",
              "                              chunk_embedding_custom  \\\n",
              "0  [0.024289679, -0.73309463, -0.37434945, 0.1869...   \n",
              "1  [-0.28205645, -0.14895901, 0.066049635, 0.5515...   \n",
              "\n",
              "   cosine_similarity_custom  id  \n",
              "0                  0.398477   0  \n",
              "1                  0.479307   1  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-e42616ed-d179-4191-929a-75d73634fcf6\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>pergunta</th>\n",
              "      <th>contexto</th>\n",
              "      <th>resposta</th>\n",
              "      <th>chunk</th>\n",
              "      <th>label</th>\n",
              "      <th>dataset</th>\n",
              "      <th>pergunta_embedding</th>\n",
              "      <th>chunk_embedding</th>\n",
              "      <th>cosine_similarity</th>\n",
              "      <th>pergunta_embedding_custom</th>\n",
              "      <th>chunk_embedding_custom</th>\n",
              "      <th>cosine_similarity_custom</th>\n",
              "      <th>id</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Qual é o segredo que o Sr. e a Sra. Dursley te...</td>\n",
              "      <td>Os Dursley tinham tudo que queriam, mas tinham...</td>\n",
              "      <td>a existência dos Potter</td>\n",
              "      <td>- CAPÍTULO UM - O menino que sobreviveu O Sr. ...</td>\n",
              "      <td>1</td>\n",
              "      <td>train</td>\n",
              "      <td>[0.022689364850521088, 0.0038326631765812635, ...</td>\n",
              "      <td>[0.014419169165194035, 0.011491919867694378, -...</td>\n",
              "      <td>0.873794</td>\n",
              "      <td>[0.17453432, -0.60518175, -0.7326583, 0.358211...</td>\n",
              "      <td>[0.024289679, -0.73309463, -0.37434945, 0.1869...</td>\n",
              "      <td>0.398477</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Qual era a cor da capa que o Sr. Dursley viu u...</td>\n",
              "      <td>O Sr. Dursley não tolerava gente que andava co...</td>\n",
              "      <td>uma capa verde-esmeralda</td>\n",
              "      <td>ante o caminho para a cidade ele não pensou em...</td>\n",
              "      <td>1</td>\n",
              "      <td>train</td>\n",
              "      <td>[0.005216388031840324, 0.0008732206770218909, ...</td>\n",
              "      <td>[-0.002306348877027631, -0.0014758666511625051...</td>\n",
              "      <td>0.873999</td>\n",
              "      <td>[-0.62348986, -0.43539473, -0.32246643, -0.106...</td>\n",
              "      <td>[-0.28205645, -0.14895901, 0.066049635, 0.5515...</td>\n",
              "      <td>0.479307</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e42616ed-d179-4191-929a-75d73634fcf6')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-e42616ed-d179-4191-929a-75d73634fcf6 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-e42616ed-d179-4191-929a-75d73634fcf6');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-edc162f7-eea5-4f82-a3a4-d6eed83977a4\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-edc162f7-eea5-4f82-a3a4-d6eed83977a4')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-edc162f7-eea5-4f82-a3a4-d6eed83977a4 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 45
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_id = pd.DataFrame(list(corpus.items()), columns=['id', 'conteudo'])\n",
        "df_id[\"pergunta\"] = df_id[\"id\"].map(topics)\n",
        "df_id.head(2)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 112
        },
        "id": "vB9BSLqb4--2",
        "outputId": "b11bc37a-c818-4d79-a87d-0c776cabb227"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   id                                           conteudo  \\\n",
              "0   0  - CAPÍTULO UM - O menino que sobreviveu O Sr. ...   \n",
              "1   1  ante o caminho para a cidade ele não pensou em...   \n",
              "\n",
              "                                            pergunta  \n",
              "0  Qual é o segredo que o Sr. e a Sra. Dursley te...  \n",
              "1  Qual era a cor da capa que o Sr. Dursley viu u...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-57e259f5-f758-477e-b9b6-d6d7aad0f9ba\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>conteudo</th>\n",
              "      <th>pergunta</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>- CAPÍTULO UM - O menino que sobreviveu O Sr. ...</td>\n",
              "      <td>Qual é o segredo que o Sr. e a Sra. Dursley te...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>ante o caminho para a cidade ele não pensou em...</td>\n",
              "      <td>Qual era a cor da capa que o Sr. Dursley viu u...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-57e259f5-f758-477e-b9b6-d6d7aad0f9ba')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-57e259f5-f758-477e-b9b6-d6d7aad0f9ba button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-57e259f5-f758-477e-b9b6-d6d7aad0f9ba');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-bd24618f-9055-4e66-9195-b78404dd58d9\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-bd24618f-9055-4e66-9195-b78404dd58d9')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-bd24618f-9055-4e66-9195-b78404dd58d9 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 46
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# pergunta:id\n",
        "map_dict = {k:v for k,v in zip(df_id[\"pergunta\"], df_id[\"id\"])}\n",
        "\n",
        "df_custom['id'] = df_custom.pergunta.apply(lambda x: map_dict[x] if x in map_dict.keys() else -1)\n",
        "\n",
        "df_ada002_custom = df_custom[(df_custom.id != -1) & (df_custom.label == 1)]\n",
        "print(len(df_ada002_custom))\n",
        "df_ada002_custom.head(2)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 199
        },
        "id": "SSryVVMA5Ikd",
        "outputId": "5f9a1112-4f8a-4779-eb5c-725b59a49434"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "140\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                            pergunta  \\\n",
              "0  Qual é o segredo que o Sr. e a Sra. Dursley te...   \n",
              "1  Qual era a cor da capa que o Sr. Dursley viu u...   \n",
              "\n",
              "                                            contexto  \\\n",
              "0  Os Dursley tinham tudo que queriam, mas tinham...   \n",
              "1  O Sr. Dursley não tolerava gente que andava co...   \n",
              "\n",
              "                   resposta  \\\n",
              "0   a existência dos Potter   \n",
              "1  uma capa verde-esmeralda   \n",
              "\n",
              "                                               chunk  label dataset  \\\n",
              "0  - CAPÍTULO UM - O menino que sobreviveu O Sr. ...      1   train   \n",
              "1  ante o caminho para a cidade ele não pensou em...      1   train   \n",
              "\n",
              "                                  pergunta_embedding  \\\n",
              "0  [0.022689364850521088, 0.0038326631765812635, ...   \n",
              "1  [0.005216388031840324, 0.0008732206770218909, ...   \n",
              "\n",
              "                                     chunk_embedding  cosine_similarity  \\\n",
              "0  [0.014419169165194035, 0.011491919867694378, -...           0.873794   \n",
              "1  [-0.002306348877027631, -0.0014758666511625051...           0.873999   \n",
              "\n",
              "                           pergunta_embedding_custom  \\\n",
              "0  [0.17453432, -0.60518175, -0.7326583, 0.358211...   \n",
              "1  [-0.62348986, -0.43539473, -0.32246643, -0.106...   \n",
              "\n",
              "                              chunk_embedding_custom  \\\n",
              "0  [0.024289679, -0.73309463, -0.37434945, 0.1869...   \n",
              "1  [-0.28205645, -0.14895901, 0.066049635, 0.5515...   \n",
              "\n",
              "   cosine_similarity_custom  id  \n",
              "0                  0.398477   0  \n",
              "1                  0.479307   1  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-1b77c2be-7e6f-4245-ab9b-d0f53e02335b\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>pergunta</th>\n",
              "      <th>contexto</th>\n",
              "      <th>resposta</th>\n",
              "      <th>chunk</th>\n",
              "      <th>label</th>\n",
              "      <th>dataset</th>\n",
              "      <th>pergunta_embedding</th>\n",
              "      <th>chunk_embedding</th>\n",
              "      <th>cosine_similarity</th>\n",
              "      <th>pergunta_embedding_custom</th>\n",
              "      <th>chunk_embedding_custom</th>\n",
              "      <th>cosine_similarity_custom</th>\n",
              "      <th>id</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Qual é o segredo que o Sr. e a Sra. Dursley te...</td>\n",
              "      <td>Os Dursley tinham tudo que queriam, mas tinham...</td>\n",
              "      <td>a existência dos Potter</td>\n",
              "      <td>- CAPÍTULO UM - O menino que sobreviveu O Sr. ...</td>\n",
              "      <td>1</td>\n",
              "      <td>train</td>\n",
              "      <td>[0.022689364850521088, 0.0038326631765812635, ...</td>\n",
              "      <td>[0.014419169165194035, 0.011491919867694378, -...</td>\n",
              "      <td>0.873794</td>\n",
              "      <td>[0.17453432, -0.60518175, -0.7326583, 0.358211...</td>\n",
              "      <td>[0.024289679, -0.73309463, -0.37434945, 0.1869...</td>\n",
              "      <td>0.398477</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Qual era a cor da capa que o Sr. Dursley viu u...</td>\n",
              "      <td>O Sr. Dursley não tolerava gente que andava co...</td>\n",
              "      <td>uma capa verde-esmeralda</td>\n",
              "      <td>ante o caminho para a cidade ele não pensou em...</td>\n",
              "      <td>1</td>\n",
              "      <td>train</td>\n",
              "      <td>[0.005216388031840324, 0.0008732206770218909, ...</td>\n",
              "      <td>[-0.002306348877027631, -0.0014758666511625051...</td>\n",
              "      <td>0.873999</td>\n",
              "      <td>[-0.62348986, -0.43539473, -0.32246643, -0.106...</td>\n",
              "      <td>[-0.28205645, -0.14895901, 0.066049635, 0.5515...</td>\n",
              "      <td>0.479307</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-1b77c2be-7e6f-4245-ab9b-d0f53e02335b')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-1b77c2be-7e6f-4245-ab9b-d0f53e02335b button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-1b77c2be-7e6f-4245-ab9b-d0f53e02335b');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-fc5ff66e-c02d-446a-9de7-fcfb72666e1d\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-fc5ff66e-c02d-446a-9de7-fcfb72666e1d')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-fc5ff66e-c02d-446a-9de7-fcfb72666e1d button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 47
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_ada002_custom[\"pergunta_embedding_custom\"].iloc[0].shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-fAVJ9zAZH7W",
        "outputId": "c0297e4f-6d58-4bfa-f2dd-6fc03e44f084"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(2048,)"
            ]
          },
          "metadata": {},
          "execution_count": 53
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def ada002_custom_search(df_ada002_custom):\n",
        "\n",
        "    query_matrix_custom = np.vstack(df_ada002_custom[\"pergunta_embedding_custom\"])\n",
        "    contents_matrix_custom = np.vstack(df_ada002_custom[\"chunk_embedding_custom\"])\n",
        "    similarity_matrix_custom = np.dot(query_matrix_custom, contents_matrix_custom.T)\n",
        "\n",
        "    ids = df_ada002_custom[\"id\"].values.tolist()\n",
        "    similarity_custom = {}\n",
        "    for query_id, hits in zip(ids, similarity_matrix_custom):\n",
        "        if similarity_custom.get(query_id) is None:\n",
        "            similarity_custom[query_id] = []\n",
        "        for doc_id, hit in zip(ids, hits):\n",
        "            similarity_custom[query_id].append((doc_id, hit))\n",
        "\n",
        "    sorted_similarity_custom = {key: sorted(\n",
        "        value, key=lambda x: x[1], reverse=True) for key, value in similarity_custom.items()}\n",
        "\n",
        "    return sorted_similarity_custom"
      ],
      "metadata": {
        "id": "90acsgfp5Ipg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sorted_similarity_custom = ada002_custom_search(df_ada002_custom)\n",
        "\n",
        "# Save run file\n",
        "experiment = \"custom_ada002\"\n",
        "run_file = f\"runs/run.hp.{experiment}.txt\"\n",
        "eval_file = f\"evals/eval.hp.{experiment}.txt\"\n",
        "\n",
        "!rm -rf $run_file\n",
        "!rm -rf $eval_file\n",
        "\n",
        "with open(run_file,'w') as f_out:\n",
        "    for query_id, hits in tqdm(sorted_similarity_custom.items(), desc=f'Writing file on {run_file}'):\n",
        "        rank = 1\n",
        "        for doc_id, score in hits:\n",
        "            f_out.write(f'{query_id}\\tQ0\\t{doc_id}\\t{rank}\\t{score}\\t{\"CUSTOM-ADA002\"}\\n')\n",
        "            rank+=1\n",
        "\n",
        "!/content/pyserini/tools/eval/trec_eval.9.0.4/trec_eval -c -M 10 -m recip_rank -m recall.3,5,7,9 qrels.tsv $run_file > $eval_file 2>&1\n",
        "trec_aux = pd.read_csv(eval_file, sep = \"\\t\", names = [\"metric\", \"all\", \"value\"])\n",
        "trec_aux[\"experiment\"] = experiment\n",
        "trec_results[experiment] = trec_aux.copy()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IxP71E045IsJ",
        "outputId": "d5fbefc1-0ef2-4dfd-c14c-a57a43f3fb47"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Writing file on runs/run.hp.custom_ada002.txt: 100%|██████████| 140/140 [00:00<00:00, 4031.41it/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "trec_results[experiment]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "ELxj9KHqXwNa",
        "outputId": "acf0f0a8-a643-4527-f275-2b2a4cdaa916"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                   metric  all   value     experiment\n",
              "0  recip_rank              all  0.6518  custom_ada002\n",
              "1  recall_3                all  0.7500  custom_ada002\n",
              "2  recall_5                all  0.8357  custom_ada002\n",
              "3  recall_7                all  0.9000  custom_ada002\n",
              "4  recall_9                all  0.9143  custom_ada002"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-ef2a8bfd-8f27-40b0-8b48-692a4e66c5a3\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>metric</th>\n",
              "      <th>all</th>\n",
              "      <th>value</th>\n",
              "      <th>experiment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>recip_rank</td>\n",
              "      <td>all</td>\n",
              "      <td>0.6518</td>\n",
              "      <td>custom_ada002</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>recall_3</td>\n",
              "      <td>all</td>\n",
              "      <td>0.7500</td>\n",
              "      <td>custom_ada002</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>recall_5</td>\n",
              "      <td>all</td>\n",
              "      <td>0.8357</td>\n",
              "      <td>custom_ada002</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>recall_7</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9000</td>\n",
              "      <td>custom_ada002</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>recall_9</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9143</td>\n",
              "      <td>custom_ada002</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-ef2a8bfd-8f27-40b0-8b48-692a4e66c5a3')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-ef2a8bfd-8f27-40b0-8b48-692a4e66c5a3 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-ef2a8bfd-8f27-40b0-8b48-692a4e66c5a3');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-3da2a157-17f5-4127-bf41-8b681c623aa0\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-3da2a157-17f5-4127-bf41-8b681c623aa0')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-3da2a157-17f5-4127-bf41-8b681c623aa0 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Hybrid Search (BM25 + ADA002-CUSTOM)"
      ],
      "metadata": {
        "id": "gkBpTGdiCMKT"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# save run file\n",
        "rrf_k_list = [0, 60]\n",
        "for rrf_k in rrf_k_list:\n",
        "    experiment = f\"hybrid_bm25_custom_ada002_k_{rrf_k}\"\n",
        "    run_file = f'runs/run.hp.{experiment}.txt'\n",
        "    eval_file = f'evals/eval.hp.{experiment}.txt'\n",
        "\n",
        "    run_file1_path = 'runs/run.hp.bm25.lucene.txt'\n",
        "    run_file2_path = 'runs/run.hp.custom_ada002.txt'\n",
        "\n",
        "    !python -m pyserini.fusion \\\n",
        "      --runs $run_file1_path $run_file2_path \\\n",
        "      --output $run_file \\\n",
        "      --rrf.k $rrf_k \\\n",
        "      --k 1000\n",
        "\n",
        "    !/content/pyserini/tools/eval/trec_eval.9.0.4/trec_eval -c -M 10 -m recip_rank -m recall.3,5,7,9 qrels.tsv $run_file > $eval_file 2>&1\n",
        "    trec_aux = pd.read_csv(eval_file, sep = \"\\t\", names = [\"metric\", \"all\", \"value\"])\n",
        "    trec_aux[\"experiment\"] = experiment\n",
        "    trec_results[experiment] = trec_aux.copy()"
      ],
      "metadata": {
        "id": "NoMGm11nKkWm"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "rrf_k_list = [0, 60]\n",
        "for rrf_k in rrf_k_list:\n",
        "    experiment = f\"hybrid_bm25_custom_ada002_k_{rrf_k}\"\n",
        "    display(trec_results[experiment])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 396
        },
        "id": "L1iNbjH6YPc1",
        "outputId": "b1fb7b3c-d7e7-4e6e-8cab-6b8424436656"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "                   metric  all   value                     experiment\n",
              "0  recip_rank              all  0.8821  hybrid_bm25_custom_ada002_k_0\n",
              "1  recall_3                all  0.9786  hybrid_bm25_custom_ada002_k_0\n",
              "2  recall_5                all  0.9929  hybrid_bm25_custom_ada002_k_0\n",
              "3  recall_7                all  0.9929  hybrid_bm25_custom_ada002_k_0\n",
              "4  recall_9                all  0.9929  hybrid_bm25_custom_ada002_k_0"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-5e0fceb3-4136-49c6-90d6-f687600bb3bb\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>metric</th>\n",
              "      <th>all</th>\n",
              "      <th>value</th>\n",
              "      <th>experiment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>recip_rank</td>\n",
              "      <td>all</td>\n",
              "      <td>0.8821</td>\n",
              "      <td>hybrid_bm25_custom_ada002_k_0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>recall_3</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9786</td>\n",
              "      <td>hybrid_bm25_custom_ada002_k_0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>recall_5</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9929</td>\n",
              "      <td>hybrid_bm25_custom_ada002_k_0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>recall_7</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9929</td>\n",
              "      <td>hybrid_bm25_custom_ada002_k_0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>recall_9</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9929</td>\n",
              "      <td>hybrid_bm25_custom_ada002_k_0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-5e0fceb3-4136-49c6-90d6-f687600bb3bb')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-5e0fceb3-4136-49c6-90d6-f687600bb3bb button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-5e0fceb3-4136-49c6-90d6-f687600bb3bb');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-f270be75-f8b6-48b4-8e26-497543051545\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-f270be75-f8b6-48b4-8e26-497543051545')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-f270be75-f8b6-48b4-8e26-497543051545 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "                   metric  all   value                      experiment\n",
              "0  recip_rank              all  0.8436  hybrid_bm25_custom_ada002_k_60\n",
              "1  recall_3                all  0.9143  hybrid_bm25_custom_ada002_k_60\n",
              "2  recall_5                all  0.9429  hybrid_bm25_custom_ada002_k_60\n",
              "3  recall_7                all  0.9714  hybrid_bm25_custom_ada002_k_60\n",
              "4  recall_9                all  0.9857  hybrid_bm25_custom_ada002_k_60"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-4b1c30ff-8476-4cb4-855b-c86c112bace1\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>metric</th>\n",
              "      <th>all</th>\n",
              "      <th>value</th>\n",
              "      <th>experiment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>recip_rank</td>\n",
              "      <td>all</td>\n",
              "      <td>0.8436</td>\n",
              "      <td>hybrid_bm25_custom_ada002_k_60</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>recall_3</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9143</td>\n",
              "      <td>hybrid_bm25_custom_ada002_k_60</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>recall_5</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9429</td>\n",
              "      <td>hybrid_bm25_custom_ada002_k_60</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>recall_7</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9714</td>\n",
              "      <td>hybrid_bm25_custom_ada002_k_60</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>recall_9</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9857</td>\n",
              "      <td>hybrid_bm25_custom_ada002_k_60</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-4b1c30ff-8476-4cb4-855b-c86c112bace1')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-4b1c30ff-8476-4cb4-855b-c86c112bace1 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-4b1c30ff-8476-4cb4-855b-c86c112bace1');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-e2473293-d7eb-46f6-8126-9aa37fdba350\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-e2473293-d7eb-46f6-8126-9aa37fdba350')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-e2473293-d7eb-46f6-8126-9aa37fdba350 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Reranker\n",
        "\n",
        "Referências:\n",
        "* https://github.com/zetaalphavector/InPars/blob/master/inpars/rerank.py\n",
        "* https://huggingface.co/unicamp-dl/mt5-base-en-pt-msmarco-v2\n",
        "* https://github.com/castorini/pygaggle/blob/master/pygaggle/rerank/transformer.py\n",
        "* https://github.com/castorini/pygaggle/blob/08339dd31f58ef40fbaa109726402e164eeba125/pygaggle/run/robust04_reranker_pipeline_gpu.py"
      ],
      "metadata": {
        "id": "4Y9o8r_-qkFp"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import csv\n",
        "import argparse\n",
        "import torch\n",
        "from math import ceil, exp\n",
        "from typing import List\n",
        "from transformers import (\n",
        "    AutoConfig,\n",
        "    AutoTokenizer,\n",
        "    AutoModelForSequenceClassification,\n",
        "    AutoModelForSeq2SeqLM,\n",
        "    T5ForConditionalGeneration,\n",
        "    T5Tokenizer,\n",
        "    MT5ForConditionalGeneration\n",
        ")"
      ],
      "metadata": {
        "id": "Afv6C5qQqjgG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "prediction_tokens = {\n",
        "        'castorini/monot5-base-msmarco':             ['▁false', '▁true'],\n",
        "        'castorini/monot5-base-msmarco-10k':         ['▁false', '▁true'],\n",
        "        'castorini/monot5-large-msmarco':            ['▁false', '▁true'],\n",
        "        'castorini/monot5-large-msmarco-10k':        ['▁false', '▁true'],\n",
        "        'castorini/monot5-base-med-msmarco':         ['▁false', '▁true'],\n",
        "        'castorini/monot5-3b-med-msmarco':           ['▁false', '▁true'],\n",
        "        'castorini/monot5-3b-msmarco-10k':           ['▁false', '▁true'],\n",
        "        'unicamp-dl/mt5-base-en-msmarco':            ['▁no'   , '▁yes'],\n",
        "        'unicamp-dl/ptt5-base-pt-msmarco-10k-v2':    ['▁não'  , '▁sim'],\n",
        "        'unicamp-dl/ptt5-base-pt-msmarco-100k-v2':   ['▁não'  , '▁sim'],\n",
        "        'unicamp-dl/ptt5-base-en-pt-msmarco-100k-v2':['▁não'  , '▁sim'],\n",
        "        'unicamp-dl/mt5-base-en-pt-msmarco-v2':      ['▁no'   , '▁yes'],\n",
        "        'unicamp-dl/mt5-base-mmarco-v2':             ['▁no'   , '▁yes'],\n",
        "        'unicamp-dl/mt5-base-en-pt-msmarco-v1':      ['▁no'   , '▁yes'],\n",
        "        'unicamp-dl/mt5-base-mmarco-v1':             ['▁no'   , '▁yes'],\n",
        "        'unicamp-dl/ptt5-base-pt-msmarco-10k-v1':    ['▁não'  , '▁sim'],\n",
        "        'unicamp-dl/ptt5-base-pt-msmarco-100k-v1':   ['▁não'  , '▁sim'],\n",
        "        'unicamp-dl/ptt5-base-en-pt-msmarco-10k-v1': ['▁não'  , '▁sim'],\n",
        "        'unicamp-dl/mt5-3B-mmarco-en-pt':            ['▁'  , '▁true'],\n",
        "        'unicamp-dl/mt5-13b-mmarco-100k':            ['▁', '▁true'],\n",
        "        }"
      ],
      "metadata": {
        "id": "barbDaSjI0y4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def chunks(lst, n):\n",
        "    for i in range(0, len(lst), n):\n",
        "        yield lst[i : i + n]"
      ],
      "metadata": {
        "id": "8MPLfXd2KY_V"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class Reranker:\n",
        "    def __init__(self, silent=False, batch_size=8, fp16=False, torchscript=False, device=None):\n",
        "        self.silent = silent\n",
        "        self.batch_size = batch_size\n",
        "        self.fp16 = fp16\n",
        "        self.torchscript = torchscript\n",
        "        self.device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "    @classmethod\n",
        "    def from_pretrained(cls, model_name_or_path, **kwargs):\n",
        "        config = AutoConfig.from_pretrained(model_name_or_path)\n",
        "        print(f\"architecture: {config.architectures}\")\n",
        "        print(f\"architecture: MonoT5Reranker\")\n",
        "        return MonoT5Reranker(model_name_or_path, **kwargs)"
      ],
      "metadata": {
        "id": "hhirFETdI01Q"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class MonoT5Reranker(Reranker):\n",
        "    name: str = 'MonoT5'\n",
        "    prompt_template: str = \"Query: {query} Document: {text} Relevant:\"\n",
        "\n",
        "    def __init__(\n",
        "        self,\n",
        "        model_name_or_path='unicamp-dl/mt5-base-en-pt-msmarco-v2',\n",
        "        token_false=None,\n",
        "        token_true=True,\n",
        "        torch_compile=False,\n",
        "        **kwargs\n",
        "    ):\n",
        "        super().__init__(**kwargs)\n",
        "        if not self.device:\n",
        "            self.device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "            print(self.device)\n",
        "        model_args = {}\n",
        "        if self.fp16:\n",
        "            model_args[\"torch_dtype\"] = torch.float16\n",
        "        self.model = MT5ForConditionalGeneration.from_pretrained(model_name_or_path, **model_args)\n",
        "        self.torch_compile = torch_compile\n",
        "        if torch_compile:\n",
        "            self.model = torch.compile(self.model)\n",
        "        self.model.to(self.device)\n",
        "        self.tokenizer = T5Tokenizer.from_pretrained(model_name_or_path)\n",
        "        self.token_false_id, self.token_true_id = self.get_prediction_tokens(\n",
        "            model_name_or_path, self.tokenizer, token_false, token_true,\n",
        "        )\n",
        "\n",
        "    def get_prediction_tokens(self, model_name_or_path, tokenizer, token_false=None, token_true=None):\n",
        "        if not (token_false and token_true):\n",
        "            if model_name_or_path in prediction_tokens:\n",
        "                token_false, token_true = prediction_tokens[model_name_or_path]\n",
        "                token_false_id = tokenizer.get_vocab()[token_false]\n",
        "                token_true_id  = tokenizer.get_vocab()[token_true]\n",
        "                return token_false_id, token_true_id\n",
        "            else:\n",
        "                # raise Exception(f\"We don't know the indexes for the non-relevant/relevant tokens for\\\n",
        "                #         the checkpoint {model_name_or_path} and you did not provide any.\")\n",
        "                return self.get_prediction_tokens('castorini/monot5-base-msmarco', self.tokenizer)\n",
        "        else:\n",
        "            token_false_id = tokenizer.get_vocab()[token_false]\n",
        "            token_true_id  = tokenizer.get_vocab()[token_true]\n",
        "            return token_false_id, token_true_id\n",
        "\n",
        "    @torch.inference_mode()\n",
        "    def rescore(self, pairs: List[List[str]]):\n",
        "        scores = []\n",
        "        for batch in tqdm(\n",
        "            chunks(pairs, self.batch_size),\n",
        "            disable=self.silent,\n",
        "            desc=\"Rescoring\",\n",
        "            total=ceil(len(pairs) / self.batch_size),\n",
        "        ):\n",
        "            prompts = [\n",
        "                self.prompt_template.format(query=query, text=text)\n",
        "                for (query, text) in batch\n",
        "            ]\n",
        "            tokens = self.tokenizer(\n",
        "                prompts,\n",
        "                padding=True,\n",
        "                truncation=True,\n",
        "                return_tensors=\"pt\",\n",
        "                max_length=self.tokenizer.model_max_length,\n",
        "                pad_to_multiple_of=(8 if self.torch_compile else None),\n",
        "            ).to(self.device)\n",
        "            output = self.model.generate(\n",
        "                **tokens,\n",
        "                max_new_tokens=1,\n",
        "                return_dict_in_generate=True,\n",
        "                output_scores=True,\n",
        "            )\n",
        "            batch_scores = output.scores[0]\n",
        "            batch_scores = batch_scores[:, [self.token_false_id, self.token_true_id]]\n",
        "            batch_scores = torch.nn.functional.log_softmax(batch_scores, dim=1)\n",
        "            scores += batch_scores[:, 1].exp().tolist()\n",
        "        return scores"
      ],
      "metadata": {
        "id": "opZPTN7tI03q"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class TRECRun:\n",
        "    def __init__(self, run_file, sep=r\"\\s+\"):\n",
        "\n",
        "        self.run_file = run_file\n",
        "        self.df = pd.read_csv(\n",
        "            run_file,\n",
        "            sep=sep,\n",
        "            quoting=csv.QUOTE_NONE,\n",
        "            keep_default_na=False,\n",
        "            names=(\"qid\", \"_1\", \"docid\", \"rank\", \"score\", \"ranker\"),\n",
        "            dtype=str,\n",
        "        )\n",
        "\n",
        "    def rerank(self, ranker, queries, corpus, top_k=1000):\n",
        "        # Converts run to float32 and subtracts a large number to ensure the BM25 scores\n",
        "        # are lower than those provided by the neural ranker.\n",
        "        self.df[\"score\"] = (\n",
        "            self.df[\"score\"]\n",
        "            .astype(\"float32\")\n",
        "            .apply(lambda x: x-10000)\n",
        "        )\n",
        "\n",
        "        # Reranks only the top-k documents for each query\n",
        "        subset = (\n",
        "            self.df[[\"qid\", \"docid\"]]\n",
        "            .groupby(\"qid\")\n",
        "            .head(top_k)\n",
        "            .apply(lambda x: [queries[int(x[\"qid\"])], corpus[int(x[\"docid\"])]], axis=1)\n",
        "        )\n",
        "        scores = ranker.rescore(subset.values.tolist())\n",
        "\n",
        "        self.df.loc[subset.index, \"score\"] = scores\n",
        "\n",
        "        self.df[\"ranker\"] = ranker.name\n",
        "        self.df = (\n",
        "            self.df\n",
        "            .groupby(\"qid\")\n",
        "            .apply(lambda x: x.sort_values(\"score\", ascending=False))\n",
        "            .reset_index(drop=True)\n",
        "        )\n",
        "\n",
        "        self.df[\"rank\"] = self.df.groupby(\"qid\").cumcount() + 1\n",
        "\n",
        "    def save(self, path):\n",
        "        self.df.to_csv(path, index=False, sep=\"\\t\", header=False, float_format='%.15f')"
      ],
      "metadata": {
        "id": "m-rqkPgfVtHW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def parse_args(args):\n",
        "    parser = argparse.ArgumentParser()\n",
        "\n",
        "    parser.add_argument(\n",
        "        \"--model\", default='castorini/monot5-small-msmarco-100k', type=str, required=False, help=\"Reranker model.\"\n",
        "        )\n",
        "    parser.add_argument(\n",
        "        \"--input_run\", default=None, type=str, help=\"Initial run to be reranked.\"\n",
        "        )\n",
        "    parser.add_argument(\n",
        "        \"--output_run\", default=None, type=str, required=True, help=\"Path to save the reranked run.\"\n",
        "        )\n",
        "    parser.add_argument(\n",
        "        \"--fp16\", default=True, type=bool, help=\"Whether to use FP16 weights during inference.\"\n",
        "        )\n",
        "    parser.add_argument(\n",
        "        \"--torch_compile\", default=True, type=bool, help=\"Whether to compile the model with `torch.compile`.\"\n",
        "        )\n",
        "    parser.add_argument(\n",
        "        \"--batch_size\", default=16, type=int, help=\"Batch size for inference.\"\n",
        "        )\n",
        "    parser.add_argument(\n",
        "        \"--top_k\", default=1_000, type=int, help=\"Top-k documents to be reranked for each query.\"\n",
        "        )\n",
        "    return parser.parse_args(args)"
      ],
      "metadata": {
        "id": "6AAmrbc5NBAe"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Save run file\n",
        "experiment = \"mt5-base-en-pt-msmarco-v2-reranker\"\n",
        "run_file = f\"runs/run.hp.{experiment}.txt\"\n",
        "eval_file = f\"evals/eval.hp.{experiment}.txt\"\n",
        "\n",
        "def dev_parser():\n",
        "    parser = parse_args([\n",
        "        '--model', 'unicamp-dl/mt5-base-en-pt-msmarco-v2',\n",
        "        '--input_run', 'runs/run.hp.bm25.lucene.txt',\n",
        "        '--output_run', run_file,\n",
        "        '--batch_size', f'16',\n",
        "        '--top_k', f'50'\n",
        "        ])\n",
        "    return parser\n",
        "\n",
        "args = dev_parser()\n",
        "print(args)\n",
        "input_run = args.input_run\n",
        "top_k = args.top_k\n",
        "output_run = args.output_run\n",
        "\n",
        "model = Reranker.from_pretrained(\n",
        "    model_name_or_path=args.model,\n",
        "    batch_size=args.batch_size,\n",
        "    fp16=args.fp16,\n",
        "    torch_compile=args.torch_compile\n",
        ")\n",
        "\n",
        "!rm -rf $run_file\n",
        "!rm -rf $eval_file\n",
        "\n",
        "run = TRECRun(input_run)\n",
        "run.rerank(model, topics, corpus, top_k=top_k)\n",
        "run.save(output_run)\n",
        "\n",
        "!/content/pyserini/tools/eval/trec_eval.9.0.4/trec_eval -c -M 10 -m recip_rank -m recall.3,5,7,9 qrels.tsv $run_file > $eval_file 2>&1\n",
        "trec_aux = pd.read_csv(eval_file, sep = \"\\t\", names = [\"metric\", \"all\", \"value\"])\n",
        "trec_aux[\"experiment\"] = experiment\n",
        "trec_results[experiment] = trec_aux.copy()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 492,
          "referenced_widgets": [
            "795b357fc8ac416a93a6e8b1f498e919",
            "78af3a65ff264a05a32a0bb0ffa5ac9f",
            "c2d655adf65f4f4182c37ae3f37952dd",
            "a1b9b277a8b24be9a21e4e4333828fa9",
            "da676b6d0c7945c59e0d47e21288441f",
            "be11f61557c8444684c0e3fdb0eed471",
            "2513b2d5ae7b48b7b24d8dbf92202370",
            "122f0038b37946cba1c3743d8ebeb18c",
            "a5847fc7deeb4d5c85791ad38fdfa4ab",
            "7ba0d616043e4b65a25f6fae499a18a3",
            "d4664ecb81b94de18c5ca259706005a9",
            "fa147a29b92741eeb30b15f2945acbf0",
            "2a44f7dbcbc04877b3d9ec43d4b2ecdf",
            "64cd4fdcd74446cfbf296331d81534f0",
            "b7d3b32c400d419e9454d752d1d57916",
            "75684a89c91f47d982213f6fd854945d",
            "7a55e65f02054dddaba6035e66f0ae6a",
            "8be94943db0a49b5a8f6d0d6dddcb8d3",
            "7e601452207b4ecea5c09f6c447d8235",
            "1e9a0cb519e04dacaeefdb57ed39f5ff",
            "d10dd697a0fe413895ee7af79366764c",
            "82fa555f26b14c8d90c0b13642de1ca9",
            "a98cec73b0a94ae7a9f35201bff15f70",
            "739261a51ef341b7ad8a26c8e8a92461",
            "06c94ffd22504439bdf0c1bfb4502ebc",
            "bc36b956045a4004a0943f1de27a2579",
            "5c109b70fc9448f2a10cd1edc7209e03",
            "f3585399a185442b9d665a43324845b9",
            "70a7c99ef3444ee3be3e19a7a2413d6d",
            "df7678ed0991463991bcd5b0b22baa30",
            "5c0b2a2cb9e44103a3eb5cb46fbf53ea",
            "ae031ed3e5b24396a79e1f03f8219ee2",
            "8d6cc27ff0184dacbc45b4bb92ff6eb8",
            "748ebb94da384dba892730915971967c",
            "ac627b7b6ed04de9897fb39af7ee5ddd",
            "d06c845863ab42f1b67ff122cb29d2d3",
            "0f964021524142e7b668ffea30d1a848",
            "1399843acb4d42418129015dfce42c99",
            "c674461cbefc4f109412e3023af74f89",
            "5b5e26ed6e514a728c276d9277f98491",
            "6a5e12fbf2054b608f021ccbe23292ef",
            "57159113b732423e915fcd930485efdc",
            "b429666c02a74abdaec5818d1c98d194",
            "df167c124cf4476f83066bbeeefcd878",
            "606b113b40e74be1abe1631fc6702ab8",
            "1621446024b14f0884ba85434c3a4f57",
            "0d2166e145a94cf7a4208b12e9a8003c",
            "80094f83acbf4cd28c378ac06316eacb",
            "c83d6f9f71664b32b33b45ab876e0a87",
            "fef0a6543a6f47c7845901eabd349d63",
            "9080c7c5781e46309c705daef7313dc6",
            "4be5d7e6cf374986a0ee0ff1079d5dee",
            "34e68c00dbec44589c357119e2448974",
            "d129c3c7a07945578194449cb6124a9c",
            "57af5665cd4241499f5c1173cb749ea8"
          ]
        },
        "id": "aMtdhcSW4wuP",
        "outputId": "d1e6c6e6-439a-4772-8214-05355d2410f5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Namespace(model='unicamp-dl/mt5-base-en-pt-msmarco-v2', input_run='runs/run.hp.bm25.lucene.txt', output_run='runs/run.hp.mt5-base-en-pt-msmarco-v2-reranker.txt', fp16=True, torch_compile=True, batch_size=16, top_k=50)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:72: UserWarning: \n",
            "The secret `HF_TOKEN` does not exist in your Colab secrets.\n",
            "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n",
            "You will be able to reuse this secret in all of your notebooks.\n",
            "Please note that authentication is recommended but still optional to access public models or datasets.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "config.json:   0%|          | 0.00/676 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "795b357fc8ac416a93a6e8b1f498e919"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "architecture: ['MT5ForConditionalGeneration']\n",
            "architecture: MonoT5Reranker\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "pytorch_model.bin:   0%|          | 0.00/2.33G [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "fa147a29b92741eeb30b15f2945acbf0"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:72: UserWarning: \n",
            "The secret `HF_TOKEN` does not exist in your Colab secrets.\n",
            "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n",
            "You will be able to reuse this secret in all of your notebooks.\n",
            "Please note that authentication is recommended but still optional to access public models or datasets.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer_config.json:   0%|          | 0.00/375 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "a98cec73b0a94ae7a9f35201bff15f70"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "spiece.model:   0%|          | 0.00/4.31M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "748ebb94da384dba892730915971967c"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "special_tokens_map.json:   0%|          | 0.00/65.0 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "606b113b40e74be1abe1631fc6702ab8"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "You are using the default legacy behaviour of the <class 'transformers.models.t5.tokenization_t5.T5Tokenizer'>. This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thouroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565\n",
            "Rescoring: 100%|██████████| 438/438 [03:28<00:00,  2.10it/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "trec_results[experiment]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "w1asixLM1pQ0",
        "outputId": "d57d7718-279e-47ac-f659-1bc76d34115d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                   metric  all   value                          experiment\n",
              "0  recip_rank              all  0.9196  mt5-base-en-pt-msmarco-v2-reranker\n",
              "1  recall_3                all  0.9714  mt5-base-en-pt-msmarco-v2-reranker\n",
              "2  recall_5                all  0.9857  mt5-base-en-pt-msmarco-v2-reranker\n",
              "3  recall_7                all  0.9929  mt5-base-en-pt-msmarco-v2-reranker\n",
              "4  recall_9                all  1.0000  mt5-base-en-pt-msmarco-v2-reranker"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-3636b389-2751-49e6-b6f0-e8d7a046d32a\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>metric</th>\n",
              "      <th>all</th>\n",
              "      <th>value</th>\n",
              "      <th>experiment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>recip_rank</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9196</td>\n",
              "      <td>mt5-base-en-pt-msmarco-v2-reranker</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>recall_3</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9714</td>\n",
              "      <td>mt5-base-en-pt-msmarco-v2-reranker</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>recall_5</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9857</td>\n",
              "      <td>mt5-base-en-pt-msmarco-v2-reranker</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>recall_7</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9929</td>\n",
              "      <td>mt5-base-en-pt-msmarco-v2-reranker</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>recall_9</td>\n",
              "      <td>all</td>\n",
              "      <td>1.0000</td>\n",
              "      <td>mt5-base-en-pt-msmarco-v2-reranker</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-3636b389-2751-49e6-b6f0-e8d7a046d32a')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-3636b389-2751-49e6-b6f0-e8d7a046d32a button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-3636b389-2751-49e6-b6f0-e8d7a046d32a');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-eae5c259-e890-4139-a903-25dd26866c93\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-eae5c259-e890-4139-a903-25dd26866c93')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-eae5c259-e890-4139-a903-25dd26866c93 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 62
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "512FOU3DvlPh"
      },
      "source": [
        "## SPLADE (TO DO)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vofR1yTavnux"
      },
      "outputs": [],
      "source": [
        "with open(\"run_splade.json\", 'r') as file:\n",
        "    run_splade = json.loads(file.read())"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Save run file\n",
        "output = f'/content/run.hp.splade.ensemble.distil.pt.txt'\n",
        "\n",
        "with open(output,'w') as f_out:\n",
        "    for query_id in tqdm(run_splade, desc=f'Writing file on {output}'):\n",
        "        rank = 1\n",
        "        hits = sorted(run_splade[query_id].items(), key=lambda x: x[1], reverse=True)\n",
        "        for doc_id, score in hits:\n",
        "            f_out.write(f'{int(query_id)}\\tQ0\\t{int(doc_id)}\\t{rank}\\t{score}\\t{\"SPLADE\"}\\n')\n",
        "            rank+=1"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PE3NSwWrxrSL",
        "outputId": "c1ff41de-b850-4adb-b9b0-7d5c7ea441eb"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Writing file on /content/run.hp.splade.ensemble.distil.pt.txt: 100%|██████████| 140/140 [00:00<00:00, 4597.00it/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!head -n 10 run.hp.splade.ensemble.distil.pt.txt"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "p6sGTMpmyYzF",
        "outputId": "2ba11176-a856-4048-ce47-4467a6320928"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0\tQ0\t0\t1\t23.49915885925293\tSPLADE\n",
            "0\tQ0\t8\t2\t18.343889236450195\tSPLADE\n",
            "0\tQ0\t2\t3\t16.050294876098633\tSPLADE\n",
            "0\tQ0\t3\t4\t15.079992294311523\tSPLADE\n",
            "0\tQ0\t10\t5\t14.750470161437988\tSPLADE\n",
            "0\tQ0\t4\t6\t14.708215713500977\tSPLADE\n",
            "0\tQ0\t1\t7\t14.512762069702148\tSPLADE\n",
            "0\tQ0\t23\t8\t14.292257308959961\tSPLADE\n",
            "0\tQ0\t12\t9\t12.748653411865234\tSPLADE\n",
            "0\tQ0\t120\t10\t12.246784210205078\tSPLADE\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!/content/pyserini/tools/eval/trec_eval.9.0.4/trec_eval -c -m map -m recip_rank -m P.1,2,3,5,10 -m recall.1,2,3,5,10 /content/qrels.tsv /content/run.hp.splade.ensemble.distil.pt.txt > eval.hp.splade.ensemble.distil.pt.txt 2>&1"
      ],
      "metadata": {
        "id": "9OhDt2vCy7o9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "y9yEtqdTo-JG"
      },
      "source": [
        "## Compile results\n",
        "\n",
        "Remember:\n",
        "* bm25.lucene == 1.0_bm25_0.0_ada002\n",
        "* ada002 == 0.0_bm25_1.0_ada002\n",
        "* bm25.lucene == 1.0_bm25_0.0_ada002_custom\n",
        "* ada002_custom == 0.0_bm25_1.0_ada002_custom"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "results = pd.DataFrame()\n",
        "for experiment in trec_results.keys():\n",
        "    results = pd.concat([results, trec_results[experiment]], axis = 0)\n",
        "\n",
        "results = results[~results[\"experiment\"].isin([\"0.0_bm25_1.0_ada002\", \"1.0_bm25_0.0_ada002\", \"0.0_bm25_1.0_ada002_custom\", \"1.0_bm25_0.0_ada002_custom\"])].copy()"
      ],
      "metadata": {
        "id": "FMsO9dcQQ3Xc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "BbkXDK92oTT9",
        "outputId": "f26ddbae-2163-49bf-c07e-7b114421f22f"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "                   metric  all   value                          experiment\n",
              "0  recip_rank              all  0.9196  mt5-base-en-pt-msmarco-v2-reranker\n",
              "0  recip_rank              all  0.8821       hybrid_bm25_custom_ada002_k_0\n",
              "0  recip_rank              all  0.8794                         bm25.lucene\n",
              "0  recip_rank              all  0.8436      hybrid_bm25_custom_ada002_k_60\n",
              "0  recip_rank              all  0.8177              hybrid_bm25_ada002_k_0\n",
              "0  recip_rank              all  0.7702             hybrid_bm25_ada002_k_60\n",
              "0  recip_rank              all  0.6518                       custom_ada002\n",
              "0  recip_rank              all  0.5839                              ada002"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-683a3c23-18c5-431d-a8b7-fcca9e02d6d7\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>metric</th>\n",
              "      <th>all</th>\n",
              "      <th>value</th>\n",
              "      <th>experiment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>recip_rank</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9196</td>\n",
              "      <td>mt5-base-en-pt-msmarco-v2-reranker</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>recip_rank</td>\n",
              "      <td>all</td>\n",
              "      <td>0.8821</td>\n",
              "      <td>hybrid_bm25_custom_ada002_k_0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>recip_rank</td>\n",
              "      <td>all</td>\n",
              "      <td>0.8794</td>\n",
              "      <td>bm25.lucene</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>recip_rank</td>\n",
              "      <td>all</td>\n",
              "      <td>0.8436</td>\n",
              "      <td>hybrid_bm25_custom_ada002_k_60</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>recip_rank</td>\n",
              "      <td>all</td>\n",
              "      <td>0.8177</td>\n",
              "      <td>hybrid_bm25_ada002_k_0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>recip_rank</td>\n",
              "      <td>all</td>\n",
              "      <td>0.7702</td>\n",
              "      <td>hybrid_bm25_ada002_k_60</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>recip_rank</td>\n",
              "      <td>all</td>\n",
              "      <td>0.6518</td>\n",
              "      <td>custom_ada002</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>recip_rank</td>\n",
              "      <td>all</td>\n",
              "      <td>0.5839</td>\n",
              "      <td>ada002</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-683a3c23-18c5-431d-a8b7-fcca9e02d6d7')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-683a3c23-18c5-431d-a8b7-fcca9e02d6d7 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-683a3c23-18c5-431d-a8b7-fcca9e02d6d7');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-6fc06a26-4c64-46db-9a79-64bb78249ced\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-6fc06a26-4c64-46db-9a79-64bb78249ced')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-6fc06a26-4c64-46db-9a79-64bb78249ced button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "                   metric  all   value                          experiment\n",
              "1  recall_3                all  0.9786       hybrid_bm25_custom_ada002_k_0\n",
              "1  recall_3                all  0.9714  mt5-base-en-pt-msmarco-v2-reranker\n",
              "1  recall_3                all  0.9429              hybrid_bm25_ada002_k_0\n",
              "1  recall_3                all  0.9143                         bm25.lucene\n",
              "1  recall_3                all  0.9143      hybrid_bm25_custom_ada002_k_60\n",
              "1  recall_3                all  0.8714             hybrid_bm25_ada002_k_60\n",
              "1  recall_3                all  0.7500                       custom_ada002\n",
              "1  recall_3                all  0.6214                              ada002"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-75487d0c-8623-4d8c-ab42-627d296b8586\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>metric</th>\n",
              "      <th>all</th>\n",
              "      <th>value</th>\n",
              "      <th>experiment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>recall_3</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9786</td>\n",
              "      <td>hybrid_bm25_custom_ada002_k_0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>recall_3</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9714</td>\n",
              "      <td>mt5-base-en-pt-msmarco-v2-reranker</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>recall_3</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9429</td>\n",
              "      <td>hybrid_bm25_ada002_k_0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>recall_3</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9143</td>\n",
              "      <td>bm25.lucene</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>recall_3</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9143</td>\n",
              "      <td>hybrid_bm25_custom_ada002_k_60</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>recall_3</td>\n",
              "      <td>all</td>\n",
              "      <td>0.8714</td>\n",
              "      <td>hybrid_bm25_ada002_k_60</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>recall_3</td>\n",
              "      <td>all</td>\n",
              "      <td>0.7500</td>\n",
              "      <td>custom_ada002</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>recall_3</td>\n",
              "      <td>all</td>\n",
              "      <td>0.6214</td>\n",
              "      <td>ada002</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-75487d0c-8623-4d8c-ab42-627d296b8586')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-75487d0c-8623-4d8c-ab42-627d296b8586 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-75487d0c-8623-4d8c-ab42-627d296b8586');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-7d10c563-1615-4b8b-9fc9-36d5216495ed\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-7d10c563-1615-4b8b-9fc9-36d5216495ed')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-7d10c563-1615-4b8b-9fc9-36d5216495ed button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "                   metric  all   value                          experiment\n",
              "2  recall_5                all  0.9929       hybrid_bm25_custom_ada002_k_0\n",
              "2  recall_5                all  0.9857  mt5-base-en-pt-msmarco-v2-reranker\n",
              "2  recall_5                all  0.9786              hybrid_bm25_ada002_k_0\n",
              "2  recall_5                all  0.9714                         bm25.lucene\n",
              "2  recall_5                all  0.9429      hybrid_bm25_custom_ada002_k_60\n",
              "2  recall_5                all  0.9143             hybrid_bm25_ada002_k_60\n",
              "2  recall_5                all  0.8357                       custom_ada002\n",
              "2  recall_5                all  0.7286                              ada002"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-01fd541f-6b82-4238-a71c-722bcf96500b\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>metric</th>\n",
              "      <th>all</th>\n",
              "      <th>value</th>\n",
              "      <th>experiment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>recall_5</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9929</td>\n",
              "      <td>hybrid_bm25_custom_ada002_k_0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>recall_5</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9857</td>\n",
              "      <td>mt5-base-en-pt-msmarco-v2-reranker</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>recall_5</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9786</td>\n",
              "      <td>hybrid_bm25_ada002_k_0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>recall_5</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9714</td>\n",
              "      <td>bm25.lucene</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>recall_5</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9429</td>\n",
              "      <td>hybrid_bm25_custom_ada002_k_60</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>recall_5</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9143</td>\n",
              "      <td>hybrid_bm25_ada002_k_60</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>recall_5</td>\n",
              "      <td>all</td>\n",
              "      <td>0.8357</td>\n",
              "      <td>custom_ada002</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>recall_5</td>\n",
              "      <td>all</td>\n",
              "      <td>0.7286</td>\n",
              "      <td>ada002</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-01fd541f-6b82-4238-a71c-722bcf96500b')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-01fd541f-6b82-4238-a71c-722bcf96500b button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-01fd541f-6b82-4238-a71c-722bcf96500b');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-c24ac6f1-8f34-4fbf-9b84-4bf14dbc2132\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-c24ac6f1-8f34-4fbf-9b84-4bf14dbc2132')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-c24ac6f1-8f34-4fbf-9b84-4bf14dbc2132 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "                   metric  all   value                          experiment\n",
              "3  recall_7                all  0.9929       hybrid_bm25_custom_ada002_k_0\n",
              "3  recall_7                all  0.9929  mt5-base-en-pt-msmarco-v2-reranker\n",
              "3  recall_7                all  0.9857                         bm25.lucene\n",
              "3  recall_7                all  0.9857              hybrid_bm25_ada002_k_0\n",
              "3  recall_7                all  0.9714      hybrid_bm25_custom_ada002_k_60\n",
              "3  recall_7                all  0.9286             hybrid_bm25_ada002_k_60\n",
              "3  recall_7                all  0.9000                       custom_ada002\n",
              "3  recall_7                all  0.8000                              ada002"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-477a7eb3-9fbf-4fd4-9c80-fd6988943349\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>metric</th>\n",
              "      <th>all</th>\n",
              "      <th>value</th>\n",
              "      <th>experiment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>recall_7</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9929</td>\n",
              "      <td>hybrid_bm25_custom_ada002_k_0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>recall_7</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9929</td>\n",
              "      <td>mt5-base-en-pt-msmarco-v2-reranker</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>recall_7</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9857</td>\n",
              "      <td>bm25.lucene</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>recall_7</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9857</td>\n",
              "      <td>hybrid_bm25_ada002_k_0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>recall_7</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9714</td>\n",
              "      <td>hybrid_bm25_custom_ada002_k_60</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>recall_7</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9286</td>\n",
              "      <td>hybrid_bm25_ada002_k_60</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>recall_7</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9000</td>\n",
              "      <td>custom_ada002</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>recall_7</td>\n",
              "      <td>all</td>\n",
              "      <td>0.8000</td>\n",
              "      <td>ada002</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-477a7eb3-9fbf-4fd4-9c80-fd6988943349')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-477a7eb3-9fbf-4fd4-9c80-fd6988943349 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-477a7eb3-9fbf-4fd4-9c80-fd6988943349');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-6b37371a-92cc-42b1-9b8d-7e2dc50d5297\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-6b37371a-92cc-42b1-9b8d-7e2dc50d5297')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-6b37371a-92cc-42b1-9b8d-7e2dc50d5297 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "                   metric  all   value                          experiment\n",
              "4  recall_9                all  1.0000  mt5-base-en-pt-msmarco-v2-reranker\n",
              "4  recall_9                all  0.9929              hybrid_bm25_ada002_k_0\n",
              "4  recall_9                all  0.9929       hybrid_bm25_custom_ada002_k_0\n",
              "4  recall_9                all  0.9857                         bm25.lucene\n",
              "4  recall_9                all  0.9857      hybrid_bm25_custom_ada002_k_60\n",
              "4  recall_9                all  0.9500             hybrid_bm25_ada002_k_60\n",
              "4  recall_9                all  0.9143                       custom_ada002\n",
              "4  recall_9                all  0.8714                              ada002"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-6522dec4-1689-4163-a752-eb2aaa63f506\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>metric</th>\n",
              "      <th>all</th>\n",
              "      <th>value</th>\n",
              "      <th>experiment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>recall_9</td>\n",
              "      <td>all</td>\n",
              "      <td>1.0000</td>\n",
              "      <td>mt5-base-en-pt-msmarco-v2-reranker</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>recall_9</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9929</td>\n",
              "      <td>hybrid_bm25_ada002_k_0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>recall_9</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9929</td>\n",
              "      <td>hybrid_bm25_custom_ada002_k_0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>recall_9</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9857</td>\n",
              "      <td>bm25.lucene</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>recall_9</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9857</td>\n",
              "      <td>hybrid_bm25_custom_ada002_k_60</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>recall_9</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9500</td>\n",
              "      <td>hybrid_bm25_ada002_k_60</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>recall_9</td>\n",
              "      <td>all</td>\n",
              "      <td>0.9143</td>\n",
              "      <td>custom_ada002</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>recall_9</td>\n",
              "      <td>all</td>\n",
              "      <td>0.8714</td>\n",
              "      <td>ada002</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-6522dec4-1689-4163-a752-eb2aaa63f506')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-6522dec4-1689-4163-a752-eb2aaa63f506 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-6522dec4-1689-4163-a752-eb2aaa63f506');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-6c4096ab-6d43-450e-8456-48139540b9a7\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-6c4096ab-6d43-450e-8456-48139540b9a7')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-6c4096ab-6d43-450e-8456-48139540b9a7 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n"
          ]
        }
      ],
      "source": [
        "metric  = 'all' #@param ['all', 'recip_rank', 'recall_3', 'recall_5', 'recall_7','recall_9'] {allow-input: true}\n",
        "\n",
        "if metric == \"all\":\n",
        "    for metric in results[\"metric\"].unique():\n",
        "        display(results[results[\"metric\"].str.strip() == metric.strip()].sort_values(by = \"value\", ascending = False))\n",
        "        print(\"\\n\")\n",
        "else:\n",
        "    display(results[results[\"metric\"].str.strip() == metric].sort_values(by = \"value\", ascending = False))"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!zip -r runs_v4.zip runs\n",
        "!zip -r evals_v4.zip evals\n",
        "\n",
        "main_dir = path.replace(\"data/\",\"\")\n",
        "shutil.copyfile(\"runs_v4.zip\", f\"{main_dir}runs_v4.zip\")\n",
        "shutil.copyfile(\"evals_v4.zip\", f\"{main_dir}evals_v4.zip\")\n",
        "print(f\"\\nSaving 'runs' and 'evals' to {main_dir}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-RPOgfWaG4G7",
        "outputId": "eabeb2ab-6d83-46df-fbb5-17a0780fd046"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "  adding: runs/ (stored 0%)\n",
            "  adding: runs/run.hp.hybrid_bm25_ada002_k_0.txt (deflated 76%)\n",
            "  adding: runs/run.hp.hybrid_bm25_custom_ada002_k_0.txt (deflated 76%)\n",
            "  adding: runs/run.hp.hybrid_bm25_custom_ada002_k_60.txt (deflated 74%)\n",
            "  adding: runs/run.hp.ada002.txt (deflated 66%)\n",
            "  adding: runs/run.hp.mt5-base-en-pt-msmarco-v2-reranker.txt (deflated 73%)\n",
            "  adding: runs/run.hp.hybrid_bm25_ada002_k_60.txt (deflated 74%)\n",
            "  adding: runs/run.hp.bm25.lucene.txt (deflated 76%)\n",
            "  adding: runs/run.hp.custom_ada002.txt (deflated 70%)\n",
            "  adding: evals/ (stored 0%)\n",
            "  adding: evals/eval.hp.ada002.txt (deflated 60%)\n",
            "  adding: evals/eval.hp.hybrid_bm25_custom_ada002_k_0.txt (deflated 66%)\n",
            "  adding: evals/eval.hp.hybrid_bm25_custom_ada002_k_60.txt (deflated 61%)\n",
            "  adding: evals/eval.hp.hybrid_bm25_ada002_k_60.txt (deflated 60%)\n",
            "  adding: evals/eval.hp.custom_ada002.txt (deflated 61%)\n",
            "  adding: evals/eval.hp.bm25.lucene.txt (deflated 64%)\n",
            "  adding: evals/eval.hp.mt5-base-en-pt-msmarco-v2-reranker.txt (deflated 60%)\n",
            "  adding: evals/eval.hp.hybrid_bm25_ada002_k_0.txt (deflated 62%)\n",
            "\n",
            "Saving 'runs' and 'evals' to /content/drive/MyDrive/22h/RAG/\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## context to LLM"
      ],
      "metadata": {
        "id": "W6zkNiGsTxvt"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "file_run_reranker    = pd.read_csv(\"runs/run.hp.mt5-base-en-pt-msmarco-v2-reranker.txt\", sep = \"\\t\", header = None, names = [\"query_id\", \"q0\", \"doc_id\", \"rank\", \"score\", \"system\"])\n",
        "file_run_hybrid      = pd.read_csv(\"runs/run.hp.hybrid_bm25_custom_ada002_k_0.txt\", sep = \"\\t\", header = None, names = [\"query_id\", \"q0\", \"doc_id\", \"rank\", \"score\", \"system\"])\n",
        "file_run_bm25        = pd.read_csv(\"runs/run.hp.bm25.lucene.txt\", sep = \" \", header = None, names = [\"query_id\", \"q0\", \"doc_id\", \"rank\", \"score\", \"system\"])\n",
        "file_run_ada_custom  = pd.read_csv(\"runs/run.hp.custom_ada002.txt\", sep = \"\\t\", header = None, names = [\"query_id\", \"q0\", \"doc_id\", \"rank\", \"score\", \"system\"])\n",
        "file_run_ada         = pd.read_csv(\"runs/run.hp.ada002.txt\", sep = \"\\t\", header = None, names = [\"query_id\", \"q0\", \"doc_id\", \"rank\", \"score\", \"system\"])"
      ],
      "metadata": {
        "id": "lvaEHn3yI8KS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def make_json_query_bestchunks(file_run, filename, json_mode=\"text\"):\n",
        "    n = [3,5,7,9]\n",
        "    for ix in n:\n",
        "\n",
        "        query_context = {}\n",
        "\n",
        "        for query_id in topics.keys():\n",
        "            hits = file_run[file_run[\"query_id\"] == query_id]\n",
        "            hits = hits.sort_values(by = \"rank\")\n",
        "            if json_mode == \"text\":\n",
        "                query_context[topics.get(query_id)] = [corpus.get(doc) for doc in list(hits[\"doc_id\"].values)[:ix]]\n",
        "            else:\n",
        "                query_context[int(query_id)] = [int(doc_id) for doc_id in list(hits[\"doc_id\"].values)[:ix]]\n",
        "\n",
        "        try:\n",
        "          file_path = f'/content/drive/MyDrive/Artigos/RAG-stuffs/data/data_retriever/{filename}_{ix}_contexts_to_llm.json'\n",
        "          with open(file_path, 'w', encoding='utf-8') as json_file:\n",
        "              json.dump(query_context, json_file, ensure_ascii=False, indent=1)\n",
        "        except:\n",
        "          file_path = f'/content/data_retriever/{filename}_{ix}_contexts_to_llm.json'\n",
        "          with open(file_path, 'w', encoding='utf-8') as json_file:\n",
        "              json.dump(query_context, json_file, ensure_ascii=False, indent=1)"
      ],
      "metadata": {
        "id": "IoSNZs5yJNAK"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "make_json_query_bestchunks(file_run=file_run_ada, filename=\"ada\")"
      ],
      "metadata": {
        "id": "MtFfUZkYJh6J"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!zip -r data_retriever.zip data_retriever\n",
        "\n",
        "main_dir = path.replace(\"data/\",\"\")\n",
        "shutil.copyfile(\"data_retriever.zip\", f\"{main_dir}data_retriever.zip\")\n",
        "print(f\"\\nSaving 'data_retriever' to {main_dir}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OS1hwgmvZ1u3",
        "outputId": "293215e8-91be-4301-9958-551c75f92b92"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "  adding: data_retriever/ (stored 0%)\n",
            "  adding: data_retriever/ada_5_contexts_to_llm.json (deflated 73%)\n",
            "  adding: data_retriever/reranker_9_contexts_to_llm.json (deflated 69%)\n",
            "  adding: data_retriever/ada_7_contexts_to_llm.json (deflated 71%)\n",
            "  adding: data_retriever/hybrid_3_contexts_to_llm.json (deflated 67%)\n",
            "  adding: data_retriever/bm25_3_contexts_to_llm.json (deflated 72%)\n",
            "  adding: data_retriever/hybrid_7_contexts_to_llm.json (deflated 67%)\n",
            "  adding: data_retriever/hybrid_5_contexts_to_llm.json (deflated 67%)\n",
            "  adding: data_retriever/reranker_5_contexts_to_llm.json (deflated 74%)\n",
            "  adding: data_retriever/ada_3_contexts_to_llm.json (deflated 75%)\n",
            "  adding: data_retriever/ada_custom_5_contexts_to_llm.json (deflated 74%)\n",
            "  adding: data_retriever/reranker_7_contexts_to_llm.json (deflated 71%)\n",
            "  adding: data_retriever/ada_custom_3_contexts_to_llm.json (deflated 74%)\n",
            "  adding: data_retriever/ada_9_contexts_to_llm.json (deflated 69%)\n",
            "  adding: data_retriever/hybrid_9_contexts_to_llm.json (deflated 67%)\n",
            "  adding: data_retriever/.ipynb_checkpoints/ (stored 0%)\n",
            "  adding: data_retriever/ada_custom_7_contexts_to_llm.json (deflated 71%)\n",
            "  adding: data_retriever/bm25_5_contexts_to_llm.json (deflated 72%)\n",
            "  adding: data_retriever/bm25_7_contexts_to_llm.json (deflated 70%)\n",
            "  adding: data_retriever/bm25_9_contexts_to_llm.json (deflated 69%)\n",
            "  adding: data_retriever/ada_custom_9_contexts_to_llm.json (deflated 69%)\n",
            "  adding: data_retriever/reranker_3_contexts_to_llm.json (deflated 75%)\n",
            "\n",
            "Saving 'data_retriever' to /content/drive/MyDrive/22h/RAG/\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "exp_json = \"reranker_3_contexts_to_llm.json\"\n",
        "try:\n",
        "    path_folder = \"/content/drive/MyDrive/Artigos/RAG-stuffs/data/data_retriever/\"\n",
        "    with open(path_folder + exp_json, 'r', encoding='utf-8') as json_file:\n",
        "        loaded_data = json.load(json_file)\n",
        "except:\n",
        "    path_folder = \"/content/data_retriever/\"\n",
        "    with open(path_folder + exp_json, 'r', encoding='utf-8') as json_file:\n",
        "        loaded_data = json.load(json_file)"
      ],
      "metadata": {
        "id": "JdvuZjy_KzAS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "qa_prompt = \"\"\"Com muita calma, responda a seguinte pergunta dentro do `contexto` que será fornecido abaixo.\n",
        "Utilize somente informações do contexto para responder.\n",
        "\n",
        "Pergunta: ```{pergunta}```\n",
        "\n",
        "\"\"\"\n",
        "\n",
        "get_context = \"\"\"\n",
        "## inicio do contexto {ix}.\n",
        "{contexto}\n",
        "## fim do contexto {ix}.\n",
        "\n",
        "###\n",
        "\"\"\"\n",
        "\n",
        "final_format = \"\"\"\\nO output deve ser ter somente a resposta.\"\"\"\n",
        "\n",
        "\n",
        "for query, chunks in loaded_data.items():\n",
        "    for chunk in chunks:\n",
        "\n",
        "        chunks_retrieved = ''\n",
        "        for ix, chunk in enumerate(chunks, 1):\n",
        "            chunks_retrieved += get_context.format(contexto=chunk, ix=ix)\n",
        "\n",
        "            if ix== len(chunks):\n",
        "                prompt = qa_prompt.format(pergunta=query) + chunks_retrieved + final_format"
      ],
      "metadata": {
        "id": "AqKCFSUaXgvg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "len(chunks)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vcGZUcoJLwIc",
        "outputId": "94733ae2-ab30-4b22-8435-91b2f7734607"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "3"
            ]
          },
          "metadata": {},
          "execution_count": 225
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(prompt)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WR2j_acpLyGq",
        "outputId": "5837a5e0-bffa-41c4-a324-fdd3e51bf8f5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Com muita calma, responda a seguinte pergunta dentro do `contexto` que será fornecido abaixo.\n",
            "Utilize somente informações do contexto para responder.\n",
            "\n",
            "Pergunta: ```Qual é a razão específica pela qual Harry Potter está sorrindo ao se despedir de Hermione e imaginar suas férias com o Duda?```\n",
            "\n",
            "\n",
            "## inicio do contexto 1.\n",
            " - Vejo vocês durante as férias, então. - Espero que você tenha... hã... umas boas férias - disse Hermione, olhando hesitante para tio Válter, espantada que alguém pudesse ser tão desagradável. - Ah, claro que sim - respondeu Harry, e eles ficaram surpresos com o sorriso que se espalhava pelo seu rosto. - Eles não sabem que não podemos fazer bruxarias em casa. Vou me divertir à beça com o Duda este verão...\n",
            "## fim do contexto 1.\n",
            "\n",
            "###\n",
            "\n",
            "## inicio do contexto 2.\n",
            " a ele não tinham mudado nem um pingo. Isto não o preocupou. Parecia-lhe que sua vida voltaria ao normal no próximo ano, ou tão normal quanto ela poderia ser em Hogwarts. Foi a melhor noite da vida de Harry, melhor do que a vitória no quadribol ou a ceia de Natal ou o encontro com o trasgo... jamais esqueceria esta noite. Harry quase esquecera que os resultados dos exames ainda estavam por vir, mas eles não deixaram de vir. Para sua grande surpresa, tanto ele quanto Rony passaram com boas notas; Hermione, é claro, foi a melhor do ano. Até Neville passou raspando, sua boa nota em Herbologia compensou a péssima nota em Poções. Tinham tido esperanças de que Goyle, que era quase tão burro quanto era mau, fosse expulso, mas ele também passou. Foi uma pena, mas, como disse Rony, não se podia ter tudo na vida.  E, de repente, seus guarda-roupas ficaram vazios, os malões arrumados, o sapo de Neville foi encontrado escondido em um canto do banheiro; as notas foram entregues a todos os alunos, com o aviso de que não fizessem bruxarias durante as férias (\"Eu sempre tenho a esperança de que eles se esqueçam de entregar as notas e o aviso\" - lamentou Fred Weasley); Hagrid estava a postos para levá-los à flotilha de barcos que fazia a travessia do lago; e, no momento seguinte, estavam embarcando no Expresso de Hogwarts; conversando e rindo à medida que os campos se tornavam mais verdes e mais cuidados; comendo feijõezinhos de todos os sabores enquanto atravessavam as cidades dos trouxas; trocando as vestes de bruxos pelos blusões e paletós; parando na plataforma nove e meia na estação de King\"s Cross. Levou um bom tempo para todos desembarcarem na plataforma. Um guarda muito velho estava postado na saída e os deixava passar em grupos de dois e três para não chamarem atenção ao irromper todos ao mesmo tempo por uma parede sólida, assustando os trouxas. - Vocês precisam vir passar uns dias conosco - disse Rony. - Os dois. Vou mandar uma coruja para vocês. - Obrigado - disse Harry. - Preciso ter alguma coisa por que esperar. As pessoas passavam empurrando-os ao se dirigirem para a saída para o mundo dos trouxas. Alguns gritavam: - Tchau, Harry! - Nos vemos por aí, Potter! - Continua famoso - comentou Rony, sorrindo para o amigo. - Não aonde eu vou, posso lhe garantir. Ele, Rony e Hermione passaram juntos pelo portão. - Olha lá ele, mamãe, olha lá ele, olha! Era Gina Weasley, a irmãzinha de Rony, mas não apontava para Rony. - Harry Potter! - gritou com a vozinha fina. - Olhe, mamãe! Estou vendo... - Fique quieta, Gina, é falta de educação apontar. A Sra. Weasley sorriu para eles. - Muito trabalho este ano? - Muito - respondeu Harry. - Obrigado pelas barrinhas de chocolate e pela suéter, Sra. Weasley. - Ah, de nada, querido. - Está pronto? Era tio Válter, ainda com a cara vermelhona, ainda bigodudo, ainda parecendo furioso com a audácia de Harry de andar carregando uma coruja numa gaiola por uma estação cheia de gente normal. Atrás dele, achavam-se tia Petúnia e Duda, parecendo aterrorizados só de olhar para Harry.  - Vocês devem ser a família de Harry! - falou a Sra. Weasley. - Por assim dizer - respondeu tio Válter. - Ande logo, menino, não temos o dia inteiro. - E se afastou. Harry ainda demorou para trocar uma última palavrinha com Rony e Hermione.\n",
            "## fim do contexto 2.\n",
            "\n",
            "###\n",
            "\n",
            "## inicio do contexto 3.\n",
            " um dia desses e em seguida se afastara sem dizer nada. A coisa mais estranha nessas pessoas era a maneira com que pareciam desaparecer no instante em que Harry tentava vê-los melhor. Na escola Harry não tinha ninguém. Todos sabiam que a turma de Duda odiava aquele estranho Harry Potter com suas roupas velhas e folgadas e os óculos remendados, e ninguém gostava de contrariar a turma do Duda.  - CAPÍTULO TRÊS - As cartas de ninguém A fuga da jiboia brasileira rendeu a Harry o seu castigo mais longo. Na altura em que lhe permitiram sair do armário, as férias de verão já haviam começado e Duda já quebrara a nova filmadora, acidentara o aeromodelo e, na primeira vez que andara na bicicleta de corrida, derrubara a velha Sra. Figg quando ela atravessava a rua dos Alfeneiros de muletas. Harry ficou contente que as aulas tivessem acabado, mas não conseguia escapar da turma de Duda, que visitava a casa todo dia. Pedro, Dênis, Malcolm e Górdon eram todos grandes e burros, mas como Duda era o maior e o mais burro do bando, era o líder. Os demais ficavam bastante felizes de participar do esporte favorito de Duda: perseguir Harry. Por esta razão Harry passava a maior parte do tempo possível fora de casa, perambulando e pensando no fim das férias, no qual conseguia vislumbrar um raiozinho de esperança. Quando setembro chegasse ele iria para a escola secundária e, pela primeira vez na vida, não estaria em companhia de Duda. Duda tinha uma vaga na antiga escola de tio Válter, Smeltings. Pedro ia para lá também. Harry, por outro lado, ia para a escola secundária local. Duda achava muita graça nisso. - Eles metem a cabeça dos garotos no vaso sanitário no primeiro dia de escola - contou ele a Harry -, quer ir lá em cima praticar? - Não, obrigado - respondeu Harry. - O coitado do vaso nunca recebeu nada tão horrível quanto a sua cabeça, é capaz de passar mal. - E correu antes que Duda conseguisse entender o que dissera. Certo dia de julho, tia Petúnia levou Duda a Londres para comprar o uniforme da Smeltings e deixou Harry com a Sra. Figg. A Sra. Figg não estava tão ruim quanto de costume. Afinal, fraturara a perna porque tropeçara em um dos gatos e não parecia gostar tanto deles quanto antes. Deixou Harry assistir à televisão e  lhe deu um pedaço de bolo de chocolate que pelo gosto parecia ter muitos anos. Naquela noite, Duda desfilou para a família reunida na sala de estar vestindo o uniforme novo da Smeltings. Os alunos da Smeltings usavam casaca marrom- avermelhada, calções cor de laranja e chapéus de palha. Carregavam também bengalas nodosas, que usavam para bater uns nos outros quando os professores não estavam olhando. Isto era considerado um bom treinamento para o futuro. Ao contemplar Duda nos calções laranja novos, tio Válter disse com a voz embargada que aquele era o momento de maior orgulho em sua vida. Tia Petúnia rompeu em lágrimas e disse que não podia acreditar que era o seu Dudinha, estava tão bonito e adulto. Harry não confiou no que poderia dizer. Achou que duas de suas costelas talvez já tivessem partido só com o esforço para não rir. Havia um cheiro horrível na cozinha na manhã seguinte quando Harry entrou para o café da manhã. Parecia vir de uma grande tina de metal dentro da pia. Ele se aproximou para espiar. A tina aparentemente estava cheia de trapos sujos que boiav\n",
            "## fim do contexto 3.\n",
            "\n",
            "###\n",
            "\n",
            "O output deve ser ter somente a resposta.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "experiment = f\"bm25.lucene\"\n",
        "run_file = f\"runs/run.hp.{experiment}.txt\"\n",
        "eval_file = f\"eval.hp.{experiment}.txt\"\n",
        "!/content/pyserini/tools/eval/trec_eval.9.0.4/trec_eval -c -M 10 -m recip_rank -m recall.3,5,7,9 qrels.tsv $run_file > $eval_file 2>&1"
      ],
      "metadata": {
        "id": "OpBzo_Xfbfne"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "experiment = f\"custom_ada002\"\n",
        "run_file = f\"runs/run.hp.{experiment}.txt\"\n",
        "eval_file = f\"eval.hp.{experiment}.txt\"\n",
        "!/content/pyserini/tools/eval/trec_eval.9.0.4/trec_eval -c -M 10 -m recip_rank -m recall.3,5,7,9 qrels.tsv $run_file > $eval_file 2>&1"
      ],
      "metadata": {
        "id": "W7s2gfMKcmEA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "experiment = f\"hybrid_bm25_custom_ada002\"\n",
        "run_file = f\"runs/run.hp.{experiment}.txt\"\n",
        "eval_file = f\"eval.hp.{experiment}.txt\"\n",
        "!/content/pyserini/tools/eval/trec_eval.9.0.4/trec_eval -c -M 10 -m recip_rank -m recall.3,5,7,9 qrels.tsv $run_file > $eval_file 2>&1"
      ],
      "metadata": {
        "id": "CjgF2IdWbnE9"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "colab": {
      "machine_shape": "hm",
      "provenance": [],
      "gpuClass": "premium",
      "collapsed_sections": [
        "512FOU3DvlPh"
      ],
      "gpuType": "V100",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "795b357fc8ac416a93a6e8b1f498e919": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_78af3a65ff264a05a32a0bb0ffa5ac9f",
              "IPY_MODEL_c2d655adf65f4f4182c37ae3f37952dd",
              "IPY_MODEL_a1b9b277a8b24be9a21e4e4333828fa9"
            ],
            "layout": "IPY_MODEL_da676b6d0c7945c59e0d47e21288441f"
          }
        },
        "78af3a65ff264a05a32a0bb0ffa5ac9f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_be11f61557c8444684c0e3fdb0eed471",
            "placeholder": "​",
            "style": "IPY_MODEL_2513b2d5ae7b48b7b24d8dbf92202370",
            "value": "config.json: 100%"
          }
        },
        "c2d655adf65f4f4182c37ae3f37952dd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_122f0038b37946cba1c3743d8ebeb18c",
            "max": 676,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_a5847fc7deeb4d5c85791ad38fdfa4ab",
            "value": 676
          }
        },
        "a1b9b277a8b24be9a21e4e4333828fa9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7ba0d616043e4b65a25f6fae499a18a3",
            "placeholder": "​",
            "style": "IPY_MODEL_d4664ecb81b94de18c5ca259706005a9",
            "value": " 676/676 [00:00&lt;00:00, 54.9kB/s]"
          }
        },
        "da676b6d0c7945c59e0d47e21288441f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "be11f61557c8444684c0e3fdb0eed471": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2513b2d5ae7b48b7b24d8dbf92202370": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "122f0038b37946cba1c3743d8ebeb18c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a5847fc7deeb4d5c85791ad38fdfa4ab": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "7ba0d616043e4b65a25f6fae499a18a3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d4664ecb81b94de18c5ca259706005a9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "fa147a29b92741eeb30b15f2945acbf0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_2a44f7dbcbc04877b3d9ec43d4b2ecdf",
              "IPY_MODEL_64cd4fdcd74446cfbf296331d81534f0",
              "IPY_MODEL_b7d3b32c400d419e9454d752d1d57916"
            ],
            "layout": "IPY_MODEL_75684a89c91f47d982213f6fd854945d"
          }
        },
        "2a44f7dbcbc04877b3d9ec43d4b2ecdf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7a55e65f02054dddaba6035e66f0ae6a",
            "placeholder": "​",
            "style": "IPY_MODEL_8be94943db0a49b5a8f6d0d6dddcb8d3",
            "value": "pytorch_model.bin: 100%"
          }
        },
        "64cd4fdcd74446cfbf296331d81534f0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7e601452207b4ecea5c09f6c447d8235",
            "max": 2329696333,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_1e9a0cb519e04dacaeefdb57ed39f5ff",
            "value": 2329696333
          }
        },
        "b7d3b32c400d419e9454d752d1d57916": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d10dd697a0fe413895ee7af79366764c",
            "placeholder": "​",
            "style": "IPY_MODEL_82fa555f26b14c8d90c0b13642de1ca9",
            "value": " 2.33G/2.33G [00:08&lt;00:00, 314MB/s]"
          }
        },
        "75684a89c91f47d982213f6fd854945d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7a55e65f02054dddaba6035e66f0ae6a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8be94943db0a49b5a8f6d0d6dddcb8d3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7e601452207b4ecea5c09f6c447d8235": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1e9a0cb519e04dacaeefdb57ed39f5ff": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "d10dd697a0fe413895ee7af79366764c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "82fa555f26b14c8d90c0b13642de1ca9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "a98cec73b0a94ae7a9f35201bff15f70": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_739261a51ef341b7ad8a26c8e8a92461",
              "IPY_MODEL_06c94ffd22504439bdf0c1bfb4502ebc",
              "IPY_MODEL_bc36b956045a4004a0943f1de27a2579"
            ],
            "layout": "IPY_MODEL_5c109b70fc9448f2a10cd1edc7209e03"
          }
        },
        "739261a51ef341b7ad8a26c8e8a92461": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f3585399a185442b9d665a43324845b9",
            "placeholder": "​",
            "style": "IPY_MODEL_70a7c99ef3444ee3be3e19a7a2413d6d",
            "value": "tokenizer_config.json: 100%"
          }
        },
        "06c94ffd22504439bdf0c1bfb4502ebc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_df7678ed0991463991bcd5b0b22baa30",
            "max": 375,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_5c0b2a2cb9e44103a3eb5cb46fbf53ea",
            "value": 375
          }
        },
        "bc36b956045a4004a0943f1de27a2579": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ae031ed3e5b24396a79e1f03f8219ee2",
            "placeholder": "​",
            "style": "IPY_MODEL_8d6cc27ff0184dacbc45b4bb92ff6eb8",
            "value": " 375/375 [00:00&lt;00:00, 30.5kB/s]"
          }
        },
        "5c109b70fc9448f2a10cd1edc7209e03": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f3585399a185442b9d665a43324845b9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "70a7c99ef3444ee3be3e19a7a2413d6d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "df7678ed0991463991bcd5b0b22baa30": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5c0b2a2cb9e44103a3eb5cb46fbf53ea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "ae031ed3e5b24396a79e1f03f8219ee2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8d6cc27ff0184dacbc45b4bb92ff6eb8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "748ebb94da384dba892730915971967c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_ac627b7b6ed04de9897fb39af7ee5ddd",
              "IPY_MODEL_d06c845863ab42f1b67ff122cb29d2d3",
              "IPY_MODEL_0f964021524142e7b668ffea30d1a848"
            ],
            "layout": "IPY_MODEL_1399843acb4d42418129015dfce42c99"
          }
        },
        "ac627b7b6ed04de9897fb39af7ee5ddd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c674461cbefc4f109412e3023af74f89",
            "placeholder": "​",
            "style": "IPY_MODEL_5b5e26ed6e514a728c276d9277f98491",
            "value": "spiece.model: 100%"
          }
        },
        "d06c845863ab42f1b67ff122cb29d2d3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6a5e12fbf2054b608f021ccbe23292ef",
            "max": 4309802,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_57159113b732423e915fcd930485efdc",
            "value": 4309802
          }
        },
        "0f964021524142e7b668ffea30d1a848": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b429666c02a74abdaec5818d1c98d194",
            "placeholder": "​",
            "style": "IPY_MODEL_df167c124cf4476f83066bbeeefcd878",
            "value": " 4.31M/4.31M [00:00&lt;00:00, 19.6MB/s]"
          }
        },
        "1399843acb4d42418129015dfce42c99": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c674461cbefc4f109412e3023af74f89": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5b5e26ed6e514a728c276d9277f98491": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6a5e12fbf2054b608f021ccbe23292ef": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "57159113b732423e915fcd930485efdc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "b429666c02a74abdaec5818d1c98d194": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "df167c124cf4476f83066bbeeefcd878": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "606b113b40e74be1abe1631fc6702ab8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_1621446024b14f0884ba85434c3a4f57",
              "IPY_MODEL_0d2166e145a94cf7a4208b12e9a8003c",
              "IPY_MODEL_80094f83acbf4cd28c378ac06316eacb"
            ],
            "layout": "IPY_MODEL_c83d6f9f71664b32b33b45ab876e0a87"
          }
        },
        "1621446024b14f0884ba85434c3a4f57": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fef0a6543a6f47c7845901eabd349d63",
            "placeholder": "​",
            "style": "IPY_MODEL_9080c7c5781e46309c705daef7313dc6",
            "value": "special_tokens_map.json: 100%"
          }
        },
        "0d2166e145a94cf7a4208b12e9a8003c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4be5d7e6cf374986a0ee0ff1079d5dee",
            "max": 65,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_34e68c00dbec44589c357119e2448974",
            "value": 65
          }
        },
        "80094f83acbf4cd28c378ac06316eacb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d129c3c7a07945578194449cb6124a9c",
            "placeholder": "​",
            "style": "IPY_MODEL_57af5665cd4241499f5c1173cb749ea8",
            "value": " 65.0/65.0 [00:00&lt;00:00, 5.85kB/s]"
          }
        },
        "c83d6f9f71664b32b33b45ab876e0a87": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fef0a6543a6f47c7845901eabd349d63": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9080c7c5781e46309c705daef7313dc6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4be5d7e6cf374986a0ee0ff1079d5dee": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "34e68c00dbec44589c357119e2448974": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "d129c3c7a07945578194449cb6124a9c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "57af5665cd4241499f5c1173cb749ea8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}