eval_mixtral.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "40c383e5-4352-477c-b972-d8d159011c9f",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Evaluate Mixtral using SageMaker Clarify\n",
    "\n",
    "See https://github.com/aws/fmeval/tree/main"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bf43d598-a3c0-4133-af3d-1edfe8e62cf7",
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install -U sagemaker"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "bf27f2a0-c14d-460b-a737-52f0a1f2a277",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: ipywidgets in /opt/conda/lib/python3.10/site-packages (7.6.5)\n",
      "Collecting ipywidgets\n",
      "  Obtaining dependency information for ipywidgets from https://files.pythonhosted.org/packages/4a/0e/57ed498fafbc60419a9332d872e929879ceba2d73cb11d284d7112472b3e/ipywidgets-8.1.1-py3-none-any.whl.metadata\n",
      "  Downloading ipywidgets-8.1.1-py3-none-any.whl.metadata (2.4 kB)\n",
      "Requirement already satisfied: comm>=0.1.3 in /opt/conda/lib/python3.10/site-packages (from ipywidgets) (0.2.0)\n",
      "Requirement already satisfied: ipython>=6.1.0 in /opt/conda/lib/python3.10/site-packages (from ipywidgets) (8.16.1)\n",
      "Requirement already satisfied: traitlets>=4.3.1 in /opt/conda/lib/python3.10/site-packages (from ipywidgets) (5.11.2)\n",
      "Collecting widgetsnbextension~=4.0.9 (from ipywidgets)\n",
      "  Obtaining dependency information for widgetsnbextension~=4.0.9 from https://files.pythonhosted.org/packages/29/03/107d96077c4befed191f7ad1a12c7b52a8f9d2778a5836d59f9855c105f6/widgetsnbextension-4.0.9-py3-none-any.whl.metadata\n",
      "  Downloading widgetsnbextension-4.0.9-py3-none-any.whl.metadata (1.6 kB)\n",
      "Collecting jupyterlab-widgets~=3.0.9 (from ipywidgets)\n",
      "  Obtaining dependency information for jupyterlab-widgets~=3.0.9 from https://files.pythonhosted.org/packages/e8/05/0ebab152288693b5ec7b339aab857362947031143b282853b4c2dd4b5b40/jupyterlab_widgets-3.0.9-py3-none-any.whl.metadata\n",
      "  Downloading jupyterlab_widgets-3.0.9-py3-none-any.whl.metadata (4.1 kB)\n",
      "Requirement already satisfied: backcall in /opt/conda/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets) (0.2.0)\n",
      "Requirement already satisfied: decorator in /opt/conda/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets) (5.1.1)\n",
      "Requirement already satisfied: jedi>=0.16 in /opt/conda/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets) (0.18.1)\n",
      "Requirement already satisfied: matplotlib-inline in /opt/conda/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets) (0.1.6)\n",
      "Requirement already satisfied: pickleshare in /opt/conda/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets) (0.7.5)\n",
      "Requirement already satisfied: prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30 in /opt/conda/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets) (3.0.39)\n",
      "Requirement already satisfied: pygments>=2.4.0 in /opt/conda/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets) (2.16.1)\n",
      "Requirement already satisfied: stack-data in /opt/conda/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets) (0.6.3)\n",
      "Requirement already satisfied: exceptiongroup in /opt/conda/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets) (1.1.3)\n",
      "Requirement already satisfied: pexpect>4.3 in /opt/conda/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets) (4.8.0)\n",
      "Requirement already satisfied: parso<0.9.0,>=0.8.0 in /opt/conda/lib/python3.10/site-packages (from jedi>=0.16->ipython>=6.1.0->ipywidgets) (0.8.3)\n",
      "Requirement already satisfied: ptyprocess>=0.5 in /opt/conda/lib/python3.10/site-packages (from pexpect>4.3->ipython>=6.1.0->ipywidgets) (0.7.0)\n",
      "Requirement already satisfied: wcwidth in /opt/conda/lib/python3.10/site-packages (from prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30->ipython>=6.1.0->ipywidgets) (0.2.5)\n",
      "Requirement already satisfied: executing>=1.2.0 in /opt/conda/lib/python3.10/site-packages (from stack-data->ipython>=6.1.0->ipywidgets) (2.0.0)\n",
      "Requirement already satisfied: asttokens>=2.1.0 in /opt/conda/lib/python3.10/site-packages (from stack-data->ipython>=6.1.0->ipywidgets) (2.4.0)\n",
      "Requirement already satisfied: pure-eval in /opt/conda/lib/python3.10/site-packages (from stack-data->ipython>=6.1.0->ipywidgets) (0.2.2)\n",
      "Requirement already satisfied: six>=1.12.0 in /opt/conda/lib/python3.10/site-packages (from asttokens>=2.1.0->stack-data->ipython>=6.1.0->ipywidgets) (1.16.0)\n",
      "Downloading ipywidgets-8.1.1-py3-none-any.whl (139 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m139.4/139.4 kB\u001b[0m \u001b[31m1.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mta \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hDownloading jupyterlab_widgets-3.0.9-py3-none-any.whl (214 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m214.9/214.9 kB\u001b[0m \u001b[31m2.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mta \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hDownloading widgetsnbextension-4.0.9-py3-none-any.whl (2.3 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.3/2.3 MB\u001b[0m \u001b[31m16.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hInstalling collected packages: widgetsnbextension, jupyterlab-widgets, ipywidgets\n",
      "  Attempting uninstall: widgetsnbextension\n",
      "    Found existing installation: widgetsnbextension 3.5.2\n",
      "    Uninstalling widgetsnbextension-3.5.2:\n",
      "      Successfully uninstalled widgetsnbextension-3.5.2\n",
      "  Attempting uninstall: jupyterlab-widgets\n",
      "    Found existing installation: jupyterlab-widgets 1.0.0\n",
      "    Uninstalling jupyterlab-widgets-1.0.0:\n",
      "      Successfully uninstalled jupyterlab-widgets-1.0.0\n",
      "  Attempting uninstall: ipywidgets\n",
      "    Found existing installation: ipywidgets 7.6.5\n",
      "    Uninstalling ipywidgets-7.6.5:\n",
      "      Successfully uninstalled ipywidgets-7.6.5\n",
      "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
      "sagemaker-datawrangler 0.4.3 requires ipywidgets<8.0.0, but you have ipywidgets 8.1.1 which is incompatible.\n",
      "sagemaker-datawrangler 0.4.3 requires sagemaker-data-insights==0.4.0, but you have sagemaker-data-insights 0.3.3 which is incompatible.\u001b[0m\u001b[31m\n",
      "\u001b[0mSuccessfully installed ipywidgets-8.1.1 jupyterlab-widgets-3.0.9 widgetsnbextension-4.0.9\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[33mWARNING: There was an error checking the latest version of pip.\u001b[0m\u001b[33m\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!pip install -U ipywidgets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8719e66e-0ae2-4610-818b-5c6059a0672a",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting fmeval\n",
      "  Obtaining dependency information for fmeval from https://files.pythonhosted.org/packages/7c/57/a5e7f98fb53e4871ca3ec91b687e9c4046802f8278cdc1c13a4fcf7193db/fmeval-0.2.1-py3-none-any.whl.metadata\n",
      "  Downloading fmeval-0.2.1-py3-none-any.whl.metadata (5.7 kB)\n",
      "Requirement already satisfied: IPython in /opt/conda/lib/python3.10/site-packages (from fmeval) (8.16.1)\n",
      "Collecting bert-score<0.4.0,>=0.3.13 (from fmeval)\n",
      "  Downloading bert_score-0.3.13-py3-none-any.whl (61 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m61.1/61.1 kB\u001b[0m \u001b[31m811.7 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hCollecting detoxify<0.6.0,>=0.5.1 (from fmeval)\n",
      "  Downloading detoxify-0.5.1-py3-none-any.whl (12 kB)\n",
      "Collecting evaluate<0.5.0,>=0.4.0 (from fmeval)\n",
      "  Obtaining dependency information for evaluate<0.5.0,>=0.4.0 from https://files.pythonhosted.org/packages/70/63/7644a1eb7b0297e585a6adec98ed9e575309bb973c33b394dae66bc35c69/evaluate-0.4.1-py3-none-any.whl.metadata\n",
      "  Downloading evaluate-0.4.1-py3-none-any.whl.metadata (9.4 kB)\n",
      "Collecting ipykernel<7.0.0,>=6.26.0 (from fmeval)\n",
      "  Obtaining dependency information for ipykernel<7.0.0,>=6.26.0 from https://files.pythonhosted.org/packages/73/14/ec9a1c42d2c952c848a1ac366718783ab725148faf910bc086601809cefd/ipykernel-6.27.1-py3-none-any.whl.metadata\n",
      "  Downloading ipykernel-6.27.1-py3-none-any.whl.metadata (6.3 kB)\n",
      "Collecting jiwer<4.0.0,>=3.0.3 (from fmeval)\n",
      "  Obtaining dependency information for jiwer<4.0.0,>=3.0.3 from https://files.pythonhosted.org/packages/0d/4f/ee537ab20144811dd99321735ff92ef2b3a3230b77ed7454bed4c44d21fc/jiwer-3.0.3-py3-none-any.whl.metadata\n",
      "  Downloading jiwer-3.0.3-py3-none-any.whl.metadata (2.6 kB)\n",
      "Requirement already satisfied: markdown in /opt/conda/lib/python3.10/site-packages (from fmeval) (3.3.4)\n",
      "Collecting matplotlib<4.0.0,>=3.8.0 (from fmeval)\n",
      "  Obtaining dependency information for matplotlib<4.0.0,>=3.8.0 from https://files.pythonhosted.org/packages/19/e5/a4ea514515f270224435c69359abb7a3d152ed31b9ee3ba5e63017461945/matplotlib-3.8.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata\n",
      "  Downloading matplotlib-3.8.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (5.8 kB)\n",
      "Collecting nltk<4.0.0,>=3.8.1 (from fmeval)\n",
      "  Downloading nltk-3.8.1-py3-none-any.whl (1.5 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.5/1.5 MB\u001b[0m \u001b[31m9.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: pandas in /opt/conda/lib/python3.10/site-packages (from fmeval) (1.4.4)\n",
      "Requirement already satisfied: pyarrow in /opt/conda/lib/python3.10/site-packages (from fmeval) (13.0.0)\n",
      "Requirement already satisfied: pyfunctional==1.4.3 in /opt/conda/lib/python3.10/site-packages (from fmeval) (1.4.3)\n",
      "Collecting ray==2.7.1 (from fmeval)\n",
      "  Obtaining dependency information for ray==2.7.1 from https://files.pythonhosted.org/packages/7d/c5/9d1ea691c4aae6abd847068861704bd2aa9167845b6591c662781f1c606a/ray-2.7.1-cp310-cp310-manylinux2014_x86_64.whl.metadata\n",
      "  Downloading ray-2.7.1-cp310-cp310-manylinux2014_x86_64.whl.metadata (13 kB)\n",
      "Collecting rouge-score<0.2.0,>=0.1.2 (from fmeval)\n",
      "  Downloading rouge_score-0.1.2.tar.gz (17 kB)\n",
      "  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hCollecting s3fs>=2023.12.0 (from fmeval)\n",
      "  Obtaining dependency information for s3fs>=2023.12.0 from https://files.pythonhosted.org/packages/5b/d6/b8a748b7d3fc7b0fd2ede1cf26a80281d65cc24d5d56b66c3a4c87e256e2/s3fs-2023.12.2-py3-none-any.whl.metadata\n",
      "  Downloading s3fs-2023.12.2-py3-none-any.whl.metadata (1.6 kB)\n",
      "Requirement already satisfied: sagemaker<3.0.0,>=2.198.0 in /opt/conda/lib/python3.10/site-packages (from fmeval) (2.199.0)\n",
      "Collecting scikit-learn<2.0.0,>=1.3.1 (from fmeval)\n",
      "  Obtaining dependency information for scikit-learn<2.0.0,>=1.3.1 from https://files.pythonhosted.org/packages/d0/0b/26ad95cf0b747be967b15fb71a06f5ac67aba0fd2f9cd174de6edefc4674/scikit_learn-1.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata\n",
      "  Downloading scikit_learn-1.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (11 kB)\n",
      "Collecting semantic-version==2.10.0 (from fmeval)\n",
      "  Downloading semantic_version-2.10.0-py2.py3-none-any.whl (15 kB)\n",
      "Collecting testbook<0.5.0,>=0.4.2 (from fmeval)\n",
      "  Downloading testbook-0.4.2-py3-none-any.whl (11 kB)\n",
      "Collecting torch!=2.0.1,!=2.1.0,>=2.0.0 (from fmeval)\n",
      "  Obtaining dependency information for torch!=2.0.1,!=2.1.0,>=2.0.0 from https://files.pythonhosted.org/packages/96/82/0966469ded5946cb4c18dd11b04eac78c943269fc79d290740d6477005e8/torch-2.1.1-cp310-cp310-manylinux1_x86_64.whl.metadata\n",
      "  Downloading torch-2.1.1-cp310-cp310-manylinux1_x86_64.whl.metadata (25 kB)\n",
      "Collecting transformers==4.22.1 (from fmeval)\n",
      "  Downloading transformers-4.22.1-py3-none-any.whl (4.9 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.9/4.9 MB\u001b[0m \u001b[31m27.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: urllib3==1.26.18 in /opt/conda/lib/python3.10/site-packages (from fmeval) (1.26.18)\n",
      "Requirement already satisfied: dill>=0.2.5 in /opt/conda/lib/python3.10/site-packages (from pyfunctional==1.4.3->fmeval) (0.3.7)\n",
      "Requirement already satisfied: tabulate<=1.0.0 in /opt/conda/lib/python3.10/site-packages (from pyfunctional==1.4.3->fmeval) (0.8.10)\n",
      "Requirement already satisfied: click>=7.0 in /opt/conda/lib/python3.10/site-packages (from ray==2.7.1->fmeval) (8.1.7)\n",
      "Requirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from ray==2.7.1->fmeval) (3.6.0)\n",
      "Requirement already satisfied: jsonschema in /opt/conda/lib/python3.10/site-packages (from ray==2.7.1->fmeval) (4.19.1)\n",
      "Requirement already satisfied: msgpack<2.0.0,>=1.0.0 in /opt/conda/lib/python3.10/site-packages (from ray==2.7.1->fmeval) (1.0.3)\n",
      "Requirement already satisfied: packaging in /opt/conda/lib/python3.10/site-packages (from ray==2.7.1->fmeval) (21.3)\n",
      "Requirement already satisfied: protobuf!=3.19.5,>=3.15.3 in /opt/conda/lib/python3.10/site-packages (from ray==2.7.1->fmeval) (4.24.4)\n",
      "Requirement already satisfied: pyyaml in /opt/conda/lib/python3.10/site-packages/PyYAML-6.0-py3.10-linux-x86_64.egg (from ray==2.7.1->fmeval) (6.0)\n",
      "Collecting aiosignal (from ray==2.7.1->fmeval)\n",
      "  Downloading aiosignal-1.3.1-py3-none-any.whl (7.6 kB)\n",
      "Collecting frozenlist (from ray==2.7.1->fmeval)\n",
      "  Obtaining dependency information for frozenlist from https://files.pythonhosted.org/packages/1e/28/74b8b6451c89c070d34e753d8b65a1e4ce508a6808b18529f36e8c0e2184/frozenlist-1.4.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata\n",
      "  Downloading frozenlist-1.4.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (5.2 kB)\n",
      "Requirement already satisfied: requests in /opt/conda/lib/python3.10/site-packages (from ray==2.7.1->fmeval) (2.31.0)\n",
      "Requirement already satisfied: numpy>=1.19.3 in /opt/conda/lib/python3.10/site-packages (from ray==2.7.1->fmeval) (1.26.0)\n",
      "Collecting huggingface-hub<1.0,>=0.9.0 (from transformers==4.22.1->fmeval)\n",
      "  Obtaining dependency information for huggingface-hub<1.0,>=0.9.0 from https://files.pythonhosted.org/packages/05/09/1945ca6ba3ad8ad6e2872ba682ce8d68c5e63c8e55458ed8ab4885709f1d/huggingface_hub-0.19.4-py3-none-any.whl.metadata\n",
      "  Downloading huggingface_hub-0.19.4-py3-none-any.whl.metadata (14 kB)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.10/site-packages (from transformers==4.22.1->fmeval) (2022.7.9)\n",
      "Collecting tokenizers!=0.11.3,<0.13,>=0.11.1 (from transformers==4.22.1->fmeval)\n",
      "  Downloading tokenizers-0.12.1-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (6.6 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.6/6.6 MB\u001b[0m \u001b[31m3.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m0:00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /opt/conda/lib/python3.10/site-packages (from transformers==4.22.1->fmeval) (4.64.1)\n",
      "Collecting sentencepiece>=0.1.94 (from detoxify<0.6.0,>=0.5.1->fmeval)\n",
      "  Downloading sentencepiece-0.1.99-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m11.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m\n",
      "\u001b[?25hCollecting datasets>=2.0.0 (from evaluate<0.5.0,>=0.4.0->fmeval)\n",
      "  Obtaining dependency information for datasets>=2.0.0 from https://files.pythonhosted.org/packages/e2/cf/db41e572d7ed958e8679018f8190438ef700aeb501b62da9e1eed9e4d69a/datasets-2.15.0-py3-none-any.whl.metadata\n",
      "  Downloading datasets-2.15.0-py3-none-any.whl.metadata (20 kB)\n",
      "Collecting xxhash (from evaluate<0.5.0,>=0.4.0->fmeval)\n",
      "  Obtaining dependency information for xxhash from https://files.pythonhosted.org/packages/80/8a/1dd41557883b6196f8f092011a5c1f72d4d44cf36d7b67d4a5efe3127949/xxhash-3.4.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata\n",
      "  Downloading xxhash-3.4.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (12 kB)\n",
      "Requirement already satisfied: multiprocess in /opt/conda/lib/python3.10/site-packages (from evaluate<0.5.0,>=0.4.0->fmeval) (0.70.15)\n",
      "Requirement already satisfied: fsspec[http]>=2021.05.0 in /opt/conda/lib/python3.10/site-packages (from evaluate<0.5.0,>=0.4.0->fmeval) (2022.7.1)\n",
      "Collecting responses<0.19 (from evaluate<0.5.0,>=0.4.0->fmeval)\n",
      "  Downloading responses-0.18.0-py3-none-any.whl (38 kB)\n",
      "Collecting comm>=0.1.1 (from ipykernel<7.0.0,>=6.26.0->fmeval)\n",
      "  Obtaining dependency information for comm>=0.1.1 from https://files.pythonhosted.org/packages/7b/a6/5fd0242e974914b139451eea0a61ed9fd2e47157e33a67939043c50a94dd/comm-0.2.0-py3-none-any.whl.metadata\n",
      "  Downloading comm-0.2.0-py3-none-any.whl.metadata (3.7 kB)\n",
      "Collecting debugpy>=1.6.5 (from ipykernel<7.0.0,>=6.26.0->fmeval)\n",
      "  Obtaining dependency information for debugpy>=1.6.5 from https://files.pythonhosted.org/packages/01/18/4be69e4b466f6452ac42b2a2cb7e581a3f1af194f1dd563d5bdabdcd8c21/debugpy-1.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata\n",
      "  Downloading debugpy-1.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (1.0 kB)\n",
      "Requirement already satisfied: jupyter-client>=6.1.12 in /opt/conda/lib/python3.10/site-packages (from ipykernel<7.0.0,>=6.26.0->fmeval) (8.4.0)\n",
      "Requirement already satisfied: jupyter-core!=5.0.*,>=4.12 in /opt/conda/lib/python3.10/site-packages (from ipykernel<7.0.0,>=6.26.0->fmeval) (5.4.0)\n",
      "Requirement already satisfied: matplotlib-inline>=0.1 in /opt/conda/lib/python3.10/site-packages (from ipykernel<7.0.0,>=6.26.0->fmeval) (0.1.6)\n",
      "Requirement already satisfied: nest-asyncio in /opt/conda/lib/python3.10/site-packages (from ipykernel<7.0.0,>=6.26.0->fmeval) (1.5.5)\n",
      "Requirement already satisfied: psutil in /opt/conda/lib/python3.10/site-packages (from ipykernel<7.0.0,>=6.26.0->fmeval) (5.9.0)\n",
      "Requirement already satisfied: pyzmq>=20 in /opt/conda/lib/python3.10/site-packages (from ipykernel<7.0.0,>=6.26.0->fmeval) (25.1.1)\n",
      "Requirement already satisfied: tornado>=6.1 in /opt/conda/lib/python3.10/site-packages (from ipykernel<7.0.0,>=6.26.0->fmeval) (6.3.3)\n",
      "Requirement already satisfied: traitlets>=5.4.0 in /opt/conda/lib/python3.10/site-packages (from ipykernel<7.0.0,>=6.26.0->fmeval) (5.11.2)\n",
      "Requirement already satisfied: backcall in /opt/conda/lib/python3.10/site-packages (from IPython->fmeval) (0.2.0)\n",
      "Requirement already satisfied: decorator in /opt/conda/lib/python3.10/site-packages (from IPython->fmeval) (5.1.1)\n",
      "Requirement already satisfied: jedi>=0.16 in /opt/conda/lib/python3.10/site-packages (from IPython->fmeval) (0.18.1)\n",
      "Requirement already satisfied: pickleshare in /opt/conda/lib/python3.10/site-packages (from IPython->fmeval) (0.7.5)\n",
      "Requirement already satisfied: prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30 in /opt/conda/lib/python3.10/site-packages (from IPython->fmeval) (3.0.39)\n",
      "Requirement already satisfied: pygments>=2.4.0 in /opt/conda/lib/python3.10/site-packages (from IPython->fmeval) (2.16.1)\n",
      "Requirement already satisfied: stack-data in /opt/conda/lib/python3.10/site-packages (from IPython->fmeval) (0.6.3)\n",
      "Requirement already satisfied: exceptiongroup in /opt/conda/lib/python3.10/site-packages (from IPython->fmeval) (1.1.3)\n",
      "Requirement already satisfied: pexpect>4.3 in /opt/conda/lib/python3.10/site-packages (from IPython->fmeval) (4.8.0)\n",
      "Collecting rapidfuzz<4,>=3 (from jiwer<4.0.0,>=3.0.3->fmeval)\n",
      "  Obtaining dependency information for rapidfuzz<4,>=3 from https://files.pythonhosted.org/packages/58/69/55f7a10f5760d8539081718f1a87172cd5b0ea21f6c1232c40a6b7cf9470/rapidfuzz-3.5.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata\n",
      "  Downloading rapidfuzz-3.5.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (11 kB)\n",
      "Requirement already satisfied: contourpy>=1.0.1 in /opt/conda/lib/python3.10/site-packages (from matplotlib<4.0.0,>=3.8.0->fmeval) (1.1.1)\n",
      "Requirement already satisfied: cycler>=0.10 in /opt/conda/lib/python3.10/site-packages (from matplotlib<4.0.0,>=3.8.0->fmeval) (0.11.0)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in /opt/conda/lib/python3.10/site-packages (from matplotlib<4.0.0,>=3.8.0->fmeval) (4.25.0)\n",
      "Requirement already satisfied: kiwisolver>=1.3.1 in /opt/conda/lib/python3.10/site-packages (from matplotlib<4.0.0,>=3.8.0->fmeval) (1.4.2)\n",
      "Requirement already satisfied: pillow>=8 in /opt/conda/lib/python3.10/site-packages (from matplotlib<4.0.0,>=3.8.0->fmeval) (10.0.1)\n",
      "Requirement already satisfied: pyparsing>=2.3.1 in /opt/conda/lib/python3.10/site-packages (from matplotlib<4.0.0,>=3.8.0->fmeval) (3.0.9)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in /opt/conda/lib/python3.10/site-packages (from matplotlib<4.0.0,>=3.8.0->fmeval) (2.8.2)\n",
      "Requirement already satisfied: joblib in /opt/conda/lib/python3.10/site-packages (from nltk<4.0.0,>=3.8.1->fmeval) (1.3.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /opt/conda/lib/python3.10/site-packages (from pandas->fmeval) (2022.1)\n",
      "Collecting absl-py (from rouge-score<0.2.0,>=0.1.2->fmeval)\n",
      "  Obtaining dependency information for absl-py from https://files.pythonhosted.org/packages/01/e4/dc0a1dcc4e74e08d7abedab278c795eef54a224363bb18f5692f416d834f/absl_py-2.0.0-py3-none-any.whl.metadata\n",
      "  Downloading absl_py-2.0.0-py3-none-any.whl.metadata (2.3 kB)\n",
      "Requirement already satisfied: six>=1.14.0 in /opt/conda/lib/python3.10/site-packages (from rouge-score<0.2.0,>=0.1.2->fmeval) (1.16.0)\n",
      "Collecting aiobotocore<3.0.0,>=2.5.4 (from s3fs>=2023.12.0->fmeval)\n",
      "  Obtaining dependency information for aiobotocore<3.0.0,>=2.5.4 from https://files.pythonhosted.org/packages/c2/28/537fc45d02e653067c5a415b6f472fcbcea3a89a3cc38111ebe0999e305c/aiobotocore-2.9.0-py3-none-any.whl.metadata\n",
      "  Downloading aiobotocore-2.9.0-py3-none-any.whl.metadata (20 kB)\n",
      "Collecting fsspec==2023.12.2 (from s3fs>=2023.12.0->fmeval)\n",
      "  Obtaining dependency information for fsspec==2023.12.2 from https://files.pythonhosted.org/packages/70/25/fab23259a52ece5670dcb8452e1af34b89e6135ecc17cd4b54b4b479eac6/fsspec-2023.12.2-py3-none-any.whl.metadata\n",
      "  Downloading fsspec-2023.12.2-py3-none-any.whl.metadata (6.8 kB)\n",
      "Collecting aiohttp!=4.0.0a0,!=4.0.0a1 (from s3fs>=2023.12.0->fmeval)\n",
      "  Obtaining dependency information for aiohttp!=4.0.0a0,!=4.0.0a1 from https://files.pythonhosted.org/packages/2f/16/50441c4baa39e5426181c6f630203ab65029f9a9c55d0a1019a31c26d702/aiohttp-3.9.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata\n",
      "  Downloading aiohttp-3.9.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (7.4 kB)\n",
      "Requirement already satisfied: attrs<24,>=23.1.0 in /opt/conda/lib/python3.10/site-packages (from sagemaker<3.0.0,>=2.198.0->fmeval) (23.1.0)\n",
      "Requirement already satisfied: boto3<2.0,>=1.33.3 in /opt/conda/lib/python3.10/site-packages (from sagemaker<3.0.0,>=2.198.0->fmeval) (1.33.12)\n",
      "Requirement already satisfied: cloudpickle==2.2.1 in /opt/conda/lib/python3.10/site-packages (from sagemaker<3.0.0,>=2.198.0->fmeval) (2.2.1)\n",
      "Requirement already satisfied: google-pasta in /opt/conda/lib/python3.10/site-packages (from sagemaker<3.0.0,>=2.198.0->fmeval) (0.2.0)\n",
      "Requirement already satisfied: smdebug-rulesconfig==1.0.1 in /opt/conda/lib/python3.10/site-packages (from sagemaker<3.0.0,>=2.198.0->fmeval) (1.0.1)\n",
      "Requirement already satisfied: importlib-metadata<7.0,>=1.4.0 in /opt/conda/lib/python3.10/site-packages (from sagemaker<3.0.0,>=2.198.0->fmeval) (4.11.3)\n",
      "Requirement already satisfied: pathos in /opt/conda/lib/python3.10/site-packages (from sagemaker<3.0.0,>=2.198.0->fmeval) (0.3.1)\n",
      "Requirement already satisfied: schema in /opt/conda/lib/python3.10/site-packages (from sagemaker<3.0.0,>=2.198.0->fmeval) (0.7.5)\n",
      "Requirement already satisfied: platformdirs in /opt/conda/lib/python3.10/site-packages (from sagemaker<3.0.0,>=2.198.0->fmeval) (2.5.2)\n",
      "Requirement already satisfied: tblib==1.7.0 in /opt/conda/lib/python3.10/site-packages (from sagemaker<3.0.0,>=2.198.0->fmeval) (1.7.0)\n",
      "Requirement already satisfied: uvicorn==0.22.0 in /opt/conda/lib/python3.10/site-packages (from sagemaker<3.0.0,>=2.198.0->fmeval) (0.22.0)\n",
      "Requirement already satisfied: fastapi==0.95.2 in /opt/conda/lib/python3.10/site-packages (from sagemaker<3.0.0,>=2.198.0->fmeval) (0.95.2)\n",
      "Requirement already satisfied: docker in /opt/conda/lib/python3.10/site-packages (from sagemaker<3.0.0,>=2.198.0->fmeval) (6.1.3)\n",
      "Requirement already satisfied: pydantic!=1.7,!=1.7.1,!=1.7.2,!=1.7.3,!=1.8,!=1.8.1,<2.0.0,>=1.6.2 in /opt/conda/lib/python3.10/site-packages (from fastapi==0.95.2->sagemaker<3.0.0,>=2.198.0->fmeval) (1.10.13)\n",
      "Requirement already satisfied: starlette<0.28.0,>=0.27.0 in /opt/conda/lib/python3.10/site-packages (from fastapi==0.95.2->sagemaker<3.0.0,>=2.198.0->fmeval) (0.27.0)\n",
      "Requirement already satisfied: h11>=0.8 in /opt/conda/lib/python3.10/site-packages (from uvicorn==0.22.0->sagemaker<3.0.0,>=2.198.0->fmeval) (0.14.0)\n",
      "Requirement already satisfied: scipy>=1.5.0 in /opt/conda/lib/python3.10/site-packages (from scikit-learn<2.0.0,>=1.3.1->fmeval) (1.11.3)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /opt/conda/lib/python3.10/site-packages (from scikit-learn<2.0.0,>=1.3.1->fmeval) (2.2.0)\n",
      "Requirement already satisfied: nbformat>=5.0.4 in /opt/conda/lib/python3.10/site-packages (from testbook<0.5.0,>=0.4.2->fmeval) (5.9.2)\n",
      "Requirement already satisfied: nbclient>=0.4.0 in /opt/conda/lib/python3.10/site-packages (from testbook<0.5.0,>=0.4.2->fmeval) (0.5.13)\n",
      "Requirement already satisfied: typing-extensions in /opt/conda/lib/python3.10/site-packages (from torch!=2.0.1,!=2.1.0,>=2.0.0->fmeval) (4.3.0)\n",
      "Requirement already satisfied: sympy in /opt/conda/lib/python3.10/site-packages (from torch!=2.0.1,!=2.1.0,>=2.0.0->fmeval) (1.10.1)\n",
      "Requirement already satisfied: networkx in /opt/conda/lib/python3.10/site-packages (from torch!=2.0.1,!=2.1.0,>=2.0.0->fmeval) (2.8.4)\n",
      "Requirement already satisfied: jinja2 in /opt/conda/lib/python3.10/site-packages (from torch!=2.0.1,!=2.1.0,>=2.0.0->fmeval) (3.1.2)\n",
      "Collecting nvidia-cuda-nvrtc-cu12==12.1.105 (from torch!=2.0.1,!=2.1.0,>=2.0.0->fmeval)\n",
      "  Downloading nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (23.7 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m23.7/23.7 MB\u001b[0m \u001b[31m27.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hCollecting nvidia-cuda-runtime-cu12==12.1.105 (from torch!=2.0.1,!=2.1.0,>=2.0.0->fmeval)\n",
      "  Downloading nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (823 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m823.6/823.6 kB\u001b[0m \u001b[31m7.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hCollecting nvidia-cuda-cupti-cu12==12.1.105 (from torch!=2.0.1,!=2.1.0,>=2.0.0->fmeval)\n",
      "  Downloading nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (14.1 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m14.1/14.1 MB\u001b[0m \u001b[31m16.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hCollecting nvidia-cudnn-cu12==8.9.2.26 (from torch!=2.0.1,!=2.1.0,>=2.0.0->fmeval)\n",
      "  Obtaining dependency information for nvidia-cudnn-cu12==8.9.2.26 from https://files.pythonhosted.org/packages/ff/74/a2e2be7fb83aaedec84f391f082cf765dfb635e7caa9b49065f73e4835d8/nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl.metadata\n",
      "  Downloading nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\n",
      "Collecting nvidia-cublas-cu12==12.1.3.1 (from torch!=2.0.1,!=2.1.0,>=2.0.0->fmeval)\n",
      "  Downloading nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl (410.6 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m410.6/410.6 MB\u001b[0m \u001b[31m2.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hCollecting nvidia-cufft-cu12==11.0.2.54 (from torch!=2.0.1,!=2.1.0,>=2.0.0->fmeval)\n",
      "  Downloading nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl (121.6 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m121.6/121.6 MB\u001b[0m \u001b[31m8.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hCollecting nvidia-curand-cu12==10.3.2.106 (from torch!=2.0.1,!=2.1.0,>=2.0.0->fmeval)\n",
      "  Downloading nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl (56.5 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.5/56.5 MB\u001b[0m \u001b[31m19.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hCollecting nvidia-cusolver-cu12==11.4.5.107 (from torch!=2.0.1,!=2.1.0,>=2.0.0->fmeval)\n",
      "  Downloading nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl (124.2 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m124.2/124.2 MB\u001b[0m \u001b[31m7.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hCollecting nvidia-cusparse-cu12==12.1.0.106 (from torch!=2.0.1,!=2.1.0,>=2.0.0->fmeval)\n",
      "  Downloading nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl (196.0 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m196.0/196.0 MB\u001b[0m \u001b[31m4.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hCollecting nvidia-nccl-cu12==2.18.1 (from torch!=2.0.1,!=2.1.0,>=2.0.0->fmeval)\n",
      "  Downloading nvidia_nccl_cu12-2.18.1-py3-none-manylinux1_x86_64.whl (209.8 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m209.8/209.8 MB\u001b[0m \u001b[31m4.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hCollecting nvidia-nvtx-cu12==12.1.105 (from torch!=2.0.1,!=2.1.0,>=2.0.0->fmeval)\n",
      "  Downloading nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (99 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m99.1/99.1 kB\u001b[0m \u001b[31m1.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mta \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hCollecting triton==2.1.0 (from torch!=2.0.1,!=2.1.0,>=2.0.0->fmeval)\n",
      "  Obtaining dependency information for triton==2.1.0 from https://files.pythonhosted.org/packages/4d/22/91a8af421c8a8902dde76e6ef3db01b258af16c53d81e8c0d0dc13900a9e/triton-2.1.0-0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata\n",
      "  Downloading triton-2.1.0-0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata (1.3 kB)\n",
      "Collecting nvidia-nvjitlink-cu12 (from nvidia-cusolver-cu12==11.4.5.107->torch!=2.0.1,!=2.1.0,>=2.0.0->fmeval)\n",
      "  Obtaining dependency information for nvidia-nvjitlink-cu12 from https://files.pythonhosted.org/packages/1e/07/bf730d44c2fe1b676ad9cc2be5f5f861eb5d153fb6951987a2d6a96379a9/nvidia_nvjitlink_cu12-12.3.101-py3-none-manylinux1_x86_64.whl.metadata\n",
      "  Downloading nvidia_nvjitlink_cu12-12.3.101-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n",
      "Requirement already satisfied: botocore<1.33.14,>=1.33.2 in /opt/conda/lib/python3.10/site-packages (from aiobotocore<3.0.0,>=2.5.4->s3fs>=2023.12.0->fmeval) (1.33.12)\n",
      "Requirement already satisfied: wrapt<2.0.0,>=1.10.10 in /opt/conda/lib/python3.10/site-packages (from aiobotocore<3.0.0,>=2.5.4->s3fs>=2023.12.0->fmeval) (1.14.1)\n",
      "Collecting aioitertools<1.0.0,>=0.5.1 (from aiobotocore<3.0.0,>=2.5.4->s3fs>=2023.12.0->fmeval)\n",
      "  Downloading aioitertools-0.11.0-py3-none-any.whl (23 kB)\n",
      "Collecting multidict<7.0,>=4.5 (from aiohttp!=4.0.0a0,!=4.0.0a1->s3fs>=2023.12.0->fmeval)\n",
      "  Downloading multidict-6.0.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (114 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m114.5/114.5 kB\u001b[0m \u001b[31m2.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hCollecting yarl<2.0,>=1.0 (from aiohttp!=4.0.0a0,!=4.0.0a1->s3fs>=2023.12.0->fmeval)\n",
      "  Obtaining dependency information for yarl<2.0,>=1.0 from https://files.pythonhosted.org/packages/c3/a0/0ade1409d184cbc9e85acd403a386a7c0563b92ff0f26d138ff9e86e48b4/yarl-1.9.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata\n",
      "  Downloading yarl-1.9.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (31 kB)\n",
      "Collecting async-timeout<5.0,>=4.0 (from aiohttp!=4.0.0a0,!=4.0.0a1->s3fs>=2023.12.0->fmeval)\n",
      "  Obtaining dependency information for async-timeout<5.0,>=4.0 from https://files.pythonhosted.org/packages/a7/fa/e01228c2938de91d47b307831c62ab9e4001e747789d0b05baf779a6488c/async_timeout-4.0.3-py3-none-any.whl.metadata\n",
      "  Downloading async_timeout-4.0.3-py3-none-any.whl.metadata (4.2 kB)\n",
      "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /opt/conda/lib/python3.10/site-packages (from boto3<2.0,>=1.33.3->sagemaker<3.0.0,>=2.198.0->fmeval) (0.10.0)\n",
      "Requirement already satisfied: s3transfer<0.9.0,>=0.8.2 in /opt/conda/lib/python3.10/site-packages (from boto3<2.0,>=1.33.3->sagemaker<3.0.0,>=2.198.0->fmeval) (0.8.2)\n",
      "Collecting pyarrow-hotfix (from datasets>=2.0.0->evaluate<0.5.0,>=0.4.0->fmeval)\n",
      "  Obtaining dependency information for pyarrow-hotfix from https://files.pythonhosted.org/packages/e4/f4/9ec2222f5f5f8ea04f66f184caafd991a39c8782e31f5b0266f101cb68ca/pyarrow_hotfix-0.6-py3-none-any.whl.metadata\n",
      "  Downloading pyarrow_hotfix-0.6-py3-none-any.whl.metadata (3.6 kB)\n",
      "Collecting fsspec[http]>=2021.05.0 (from evaluate<0.5.0,>=0.4.0->fmeval)\n",
      "  Obtaining dependency information for fsspec[http]>=2021.05.0 from https://files.pythonhosted.org/packages/e8/f6/3eccfb530aac90ad1301c582da228e4763f19e719ac8200752a4841b0b2d/fsspec-2023.10.0-py3-none-any.whl.metadata\n",
      "  Downloading fsspec-2023.10.0-py3-none-any.whl.metadata (6.8 kB)\n",
      "INFO: pip is looking at multiple versions of fsspec[http] to determine which version is compatible with other requirements. This could take a while.\n",
      "  Obtaining dependency information for fsspec[http]>=2021.05.0 from https://files.pythonhosted.org/packages/fe/d3/e1aa96437d944fbb9cc95d0316e25583886e9cd9e6adc07baad943524eda/fsspec-2023.9.2-py3-none-any.whl.metadata\n",
      "  Downloading fsspec-2023.9.2-py3-none-any.whl.metadata (6.7 kB)\n",
      "  Obtaining dependency information for fsspec[http]>=2021.05.0 from https://files.pythonhosted.org/packages/6a/af/c673e8c663e17bd4fb201a6f029153ad5d7023aa4442d81c7987743db379/fsspec-2023.9.1-py3-none-any.whl.metadata\n",
      "  Downloading fsspec-2023.9.1-py3-none-any.whl.metadata (6.7 kB)\n",
      "  Obtaining dependency information for fsspec[http]>=2021.05.0 from https://files.pythonhosted.org/packages/3a/9f/b40e8e5be886143379000af5fc0c675352d59e82fd869d24bf784161dc77/fsspec-2023.9.0-py3-none-any.whl.metadata\n",
      "  Downloading fsspec-2023.9.0-py3-none-any.whl.metadata (6.7 kB)\n",
      "  Obtaining dependency information for fsspec[http]>=2021.05.0 from https://files.pythonhosted.org/packages/e3/bd/4c0a4619494188a9db5d77e2100ab7d544a42e76b2447869d8e124e981d8/fsspec-2023.6.0-py3-none-any.whl.metadata\n",
      "  Downloading fsspec-2023.6.0-py3-none-any.whl.metadata (6.7 kB)\n",
      "  Downloading fsspec-2023.5.0-py3-none-any.whl (160 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m160.1/160.1 kB\u001b[0m \u001b[31m2.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25h  Downloading fsspec-2023.4.0-py3-none-any.whl (153 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m154.0/154.0 kB\u001b[0m \u001b[31m2.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25h  Downloading fsspec-2023.3.0-py3-none-any.whl (145 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m145.4/145.4 kB\u001b[0m \u001b[31m2.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hINFO: pip is still looking at multiple versions of fsspec[http] to determine which version is compatible with other requirements. This could take a while.\n",
      "  Downloading fsspec-2023.1.0-py3-none-any.whl (143 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m143.0/143.0 kB\u001b[0m \u001b[31m2.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hCollecting datasets>=2.0.0 (from evaluate<0.5.0,>=0.4.0->fmeval)\n",
      "  Obtaining dependency information for datasets>=2.0.0 from https://files.pythonhosted.org/packages/00/23/80a2147a547cb2fd59eb92a13787c849b3efaefcea02a5c963dfc93f7c56/datasets-2.14.7-py3-none-any.whl.metadata\n",
      "  Downloading datasets-2.14.7-py3-none-any.whl.metadata (19 kB)\n",
      "INFO: pip is looking at multiple versions of datasets to determine which version is compatible with other requirements. This could take a while.\n",
      "  Obtaining dependency information for datasets>=2.0.0 from https://files.pythonhosted.org/packages/7c/55/b3432f43d6d7fee999bb23a547820d74c48ec540f5f7842e41aa5d8d5f3a/datasets-2.14.6-py3-none-any.whl.metadata\n",
      "  Downloading datasets-2.14.6-py3-none-any.whl.metadata (19 kB)\n",
      "  Obtaining dependency information for datasets>=2.0.0 from https://files.pythonhosted.org/packages/09/7e/fd4d6441a541dba61d0acb3c1fd5df53214c2e9033854e837a99dd9e0793/datasets-2.14.5-py3-none-any.whl.metadata\n",
      "  Downloading datasets-2.14.5-py3-none-any.whl.metadata (19 kB)\n",
      "  Obtaining dependency information for datasets>=2.0.0 from https://files.pythonhosted.org/packages/66/f8/38298237d18d4b6a8ee5dfe390e97bed5adb8e01ec6f9680c0ddf3066728/datasets-2.14.4-py3-none-any.whl.metadata\n",
      "  Downloading datasets-2.14.4-py3-none-any.whl.metadata (19 kB)\n",
      "Requirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.10/site-packages (from importlib-metadata<7.0,>=1.4.0->sagemaker<3.0.0,>=2.198.0->fmeval) (3.8.0)\n",
      "Requirement already satisfied: parso<0.9.0,>=0.8.0 in /opt/conda/lib/python3.10/site-packages (from jedi>=0.16->IPython->fmeval) (0.8.3)\n",
      "Requirement already satisfied: fastjsonschema in /opt/conda/lib/python3.10/site-packages (from nbformat>=5.0.4->testbook<0.5.0,>=0.4.2->fmeval) (2.16.2)\n",
      "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /opt/conda/lib/python3.10/site-packages (from jsonschema->ray==2.7.1->fmeval) (2023.7.1)\n",
      "Requirement already satisfied: referencing>=0.28.4 in /opt/conda/lib/python3.10/site-packages (from jsonschema->ray==2.7.1->fmeval) (0.30.2)\n",
      "Requirement already satisfied: rpds-py>=0.7.1 in /opt/conda/lib/python3.10/site-packages (from jsonschema->ray==2.7.1->fmeval) (0.10.6)\n",
      "Requirement already satisfied: ptyprocess>=0.5 in /opt/conda/lib/python3.10/site-packages (from pexpect>4.3->IPython->fmeval) (0.7.0)\n",
      "Requirement already satisfied: wcwidth in /opt/conda/lib/python3.10/site-packages (from prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30->IPython->fmeval) (0.2.5)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests->ray==2.7.1->fmeval) (2.0.4)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests->ray==2.7.1->fmeval) (3.3)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests->ray==2.7.1->fmeval) (2023.7.22)\n",
      "Requirement already satisfied: websocket-client>=0.32.0 in /opt/conda/lib/python3.10/site-packages (from docker->sagemaker<3.0.0,>=2.198.0->fmeval) (0.58.0)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.10/site-packages (from jinja2->torch!=2.0.1,!=2.1.0,>=2.0.0->fmeval) (2.1.3)\n",
      "Requirement already satisfied: ppft>=1.7.6.7 in /opt/conda/lib/python3.10/site-packages (from pathos->sagemaker<3.0.0,>=2.198.0->fmeval) (1.7.6.7)\n",
      "Requirement already satisfied: pox>=0.3.3 in /opt/conda/lib/python3.10/site-packages (from pathos->sagemaker<3.0.0,>=2.198.0->fmeval) (0.3.3)\n",
      "Requirement already satisfied: contextlib2>=0.5.5 in /opt/conda/lib/python3.10/site-packages (from schema->sagemaker<3.0.0,>=2.198.0->fmeval) (21.6.0)\n",
      "Requirement already satisfied: executing>=1.2.0 in /opt/conda/lib/python3.10/site-packages (from stack-data->IPython->fmeval) (2.0.0)\n",
      "Requirement already satisfied: asttokens>=2.1.0 in /opt/conda/lib/python3.10/site-packages (from stack-data->IPython->fmeval) (2.4.0)\n",
      "Requirement already satisfied: pure-eval in /opt/conda/lib/python3.10/site-packages (from stack-data->IPython->fmeval) (0.2.2)\n",
      "Requirement already satisfied: mpmath>=0.19 in /opt/conda/lib/python3.10/site-packages (from sympy->torch!=2.0.1,!=2.1.0,>=2.0.0->fmeval) (1.3.0)\n",
      "Requirement already satisfied: anyio<5,>=3.4.0 in /opt/conda/lib/python3.10/site-packages (from starlette<0.28.0,>=0.27.0->fastapi==0.95.2->sagemaker<3.0.0,>=2.198.0->fmeval) (3.5.0)\n",
      "Requirement already satisfied: sniffio>=1.1 in /opt/conda/lib/python3.10/site-packages (from anyio<5,>=3.4.0->starlette<0.28.0,>=0.27.0->fastapi==0.95.2->sagemaker<3.0.0,>=2.198.0->fmeval) (1.2.0)\n",
      "Downloading fmeval-0.2.1-py3-none-any.whl (103 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m103.1/103.1 kB\u001b[0m \u001b[31m2.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m\n",
      "\u001b[?25hDownloading ray-2.7.1-cp310-cp310-manylinux2014_x86_64.whl (62.4 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.4/62.4 MB\u001b[0m \u001b[31m16.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hDownloading evaluate-0.4.1-py3-none-any.whl (84 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m84.1/84.1 kB\u001b[0m \u001b[31m1.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mta \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hDownloading ipykernel-6.27.1-py3-none-any.whl (114 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m114.6/114.6 kB\u001b[0m \u001b[31m1.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m\n",
      "\u001b[?25hDownloading jiwer-3.0.3-py3-none-any.whl (21 kB)\n",
      "Downloading matplotlib-3.8.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (11.6 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m11.6/11.6 MB\u001b[0m \u001b[31m43.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hDownloading s3fs-2023.12.2-py3-none-any.whl (28 kB)\n",
      "Downloading fsspec-2023.12.2-py3-none-any.whl (168 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m169.0/169.0 kB\u001b[0m \u001b[31m2.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mta \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hDownloading scikit_learn-1.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (10.8 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m10.8/10.8 MB\u001b[0m \u001b[31m42.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m0:01\u001b[0m\n",
      "\u001b[?25hDownloading torch-2.1.1-cp310-cp310-manylinux1_x86_64.whl (670.2 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m670.2/670.2 MB\u001b[0m \u001b[31m1.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hDownloading nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl (731.7 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m731.7/731.7 MB\u001b[0m \u001b[31m1,000.0 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hDownloading triton-2.1.0-0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (89.2 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m89.2/89.2 MB\u001b[0m \u001b[31m11.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hDownloading aiobotocore-2.9.0-py3-none-any.whl (75 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m76.0/76.0 kB\u001b[0m \u001b[31m1.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0meta \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hDownloading aiohttp-3.9.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.2 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m14.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m\n",
      "\u001b[?25hDownloading comm-0.2.0-py3-none-any.whl (7.0 kB)\n",
      "Downloading datasets-2.14.4-py3-none-any.whl (519 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m519.3/519.3 kB\u001b[0m \u001b[31m7.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m\n",
      "\u001b[?25hDownloading debugpy-1.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.3 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.3/3.3 MB\u001b[0m \u001b[31m24.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hDownloading frozenlist-1.4.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (225 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m225.7/225.7 kB\u001b[0m \u001b[31m2.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m\n",
      "\u001b[?25hDownloading huggingface_hub-0.19.4-py3-none-any.whl (311 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m311.7/311.7 kB\u001b[0m \u001b[31m5.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m\n",
      "\u001b[?25hDownloading rapidfuzz-3.5.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.3 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.3/3.3 MB\u001b[0m \u001b[31m16.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hDownloading absl_py-2.0.0-py3-none-any.whl (130 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m130.2/130.2 kB\u001b[0m \u001b[31m2.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mta \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hDownloading xxhash-3.4.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (194 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m194.1/194.1 kB\u001b[0m \u001b[31m3.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mta \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hDownloading async_timeout-4.0.3-py3-none-any.whl (5.7 kB)\n",
      "Downloading yarl-1.9.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (301 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m301.6/301.6 kB\u001b[0m \u001b[31m5.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m\n",
      "\u001b[?25hDownloading nvidia_nvjitlink_cu12-12.3.101-py3-none-manylinux1_x86_64.whl (20.5 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m20.5/20.5 MB\u001b[0m \u001b[31m28.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hBuilding wheels for collected packages: rouge-score\n",
      "  Building wheel for rouge-score (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for rouge-score: filename=rouge_score-0.1.2-py3-none-any.whl size=24932 sha256=8a85cb58c8620480d1f1c5846603d09c4bac51db29ccf93ae9da613aef585ea9\n",
      "  Stored in directory: /root/.cache/pip/wheels/5f/dd/89/461065a73be61a532ff8599a28e9beef17985c9e9c31e541b4\n",
      "Successfully built rouge-score\n",
      "Installing collected packages: tokenizers, sentencepiece, xxhash, triton, semantic-version, rapidfuzz, nvidia-nvtx-cu12, nvidia-nvjitlink-cu12, nvidia-nccl-cu12, nvidia-curand-cu12, nvidia-cufft-cu12, nvidia-cuda-runtime-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-cupti-cu12, nvidia-cublas-cu12, nltk, multidict, fsspec, frozenlist, debugpy, comm, async-timeout, aioitertools, absl-py, yarl, scikit-learn, rouge-score, responses, nvidia-cusparse-cu12, nvidia-cudnn-cu12, matplotlib, jiwer, huggingface-hub, aiosignal, transformers, nvidia-cusolver-cu12, aiohttp, torch, ray, ipykernel, aiobotocore, s3fs, detoxify, datasets, bert-score, testbook, evaluate, fmeval\n",
      "  Attempting uninstall: nltk\n",
      "    Found existing installation: nltk 3.7\n",
      "    Uninstalling nltk-3.7:\n",
      "      Successfully uninstalled nltk-3.7\n",
      "  Attempting uninstall: fsspec\n",
      "    Found existing installation: fsspec 2022.7.1\n",
      "    Uninstalling fsspec-2022.7.1:\n",
      "      Successfully uninstalled fsspec-2022.7.1\n",
      "  Attempting uninstall: debugpy\n",
      "    Found existing installation: debugpy 1.5.1\n",
      "    Uninstalling debugpy-1.5.1:\n",
      "      Successfully uninstalled debugpy-1.5.1\n",
      "  Attempting uninstall: scikit-learn\n",
      "    Found existing installation: scikit-learn 1.0.1\n",
      "    Uninstalling scikit-learn-1.0.1:\n",
      "      Successfully uninstalled scikit-learn-1.0.1\n",
      "  Attempting uninstall: matplotlib\n",
      "    Found existing installation: matplotlib 3.5.2\n",
      "    Uninstalling matplotlib-3.5.2:\n",
      "      Successfully uninstalled matplotlib-3.5.2\n",
      "  Attempting uninstall: ipykernel\n",
      "    Found existing installation: ipykernel 6.15.0\n",
      "    Uninstalling ipykernel-6.15.0:\n",
      "      Successfully uninstalled ipykernel-6.15.0\n",
      "  Attempting uninstall: s3fs\n",
      "    Found existing installation: s3fs 0.4.2\n",
      "    Uninstalling s3fs-0.4.2:\n",
      "      Successfully uninstalled s3fs-0.4.2\n",
      "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
      "spyder 5.3.3 requires pyqt5<5.16, which is not installed.\n",
      "spyder 5.3.3 requires pyqtwebengine<5.16, which is not installed.\n",
      "distributed 2022.7.0 requires tornado<6.2,>=6.0.3, but you have tornado 6.3.3 which is incompatible.\n",
      "notebook 6.5.6 requires jupyter-client<8,>=5.3.4, but you have jupyter-client 8.4.0 which is incompatible.\n",
      "notebook 6.5.6 requires pyzmq<25,>=17, but you have pyzmq 25.1.1 which is incompatible.\n",
      "sagemaker-datawrangler 0.4.3 requires sagemaker-data-insights==0.4.0, but you have sagemaker-data-insights 0.3.3 which is incompatible.\n",
      "spyder 5.3.3 requires ipython<8.0.0,>=7.31.1, but you have ipython 8.16.1 which is incompatible.\n",
      "spyder 5.3.3 requires pylint<3.0,>=2.5.0, but you have pylint 3.0.1 which is incompatible.\n",
      "spyder-kernels 2.3.3 requires ipython<8,>=7.31.1; python_version >= \"3\", but you have ipython 8.16.1 which is incompatible.\n",
      "spyder-kernels 2.3.3 requires jupyter-client<8,>=7.3.4; python_version >= \"3\", but you have jupyter-client 8.4.0 which is incompatible.\u001b[0m\u001b[31m\n",
      "\u001b[0mSuccessfully installed absl-py-2.0.0 aiobotocore-2.9.0 aiohttp-3.9.1 aioitertools-0.11.0 aiosignal-1.3.1 async-timeout-4.0.3 bert-score-0.3.13 comm-0.2.0 datasets-2.14.4 debugpy-1.8.0 detoxify-0.5.1 evaluate-0.4.1 fmeval-0.2.1 frozenlist-1.4.0 fsspec-2023.12.2 huggingface-hub-0.19.4 ipykernel-6.27.1 jiwer-3.0.3 matplotlib-3.8.2 multidict-6.0.4 nltk-3.8.1 nvidia-cublas-cu12-12.1.3.1 nvidia-cuda-cupti-cu12-12.1.105 nvidia-cuda-nvrtc-cu12-12.1.105 nvidia-cuda-runtime-cu12-12.1.105 nvidia-cudnn-cu12-8.9.2.26 nvidia-cufft-cu12-11.0.2.54 nvidia-curand-cu12-10.3.2.106 nvidia-cusolver-cu12-11.4.5.107 nvidia-cusparse-cu12-12.1.0.106 nvidia-nccl-cu12-2.18.1 nvidia-nvjitlink-cu12-12.3.101 nvidia-nvtx-cu12-12.1.105 rapidfuzz-3.5.2 ray-2.7.1 responses-0.18.0 rouge-score-0.1.2 s3fs-2023.12.2 scikit-learn-1.3.2 semantic-version-2.10.0 sentencepiece-0.1.99 testbook-0.4.2 tokenizers-0.12.1 torch-2.1.1 transformers-4.22.1 triton-2.1.0 xxhash-3.4.1 yarl-1.9.4\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[33mWARNING: There was an error checking the latest version of pip.\u001b[0m\u001b[33m\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!pip install fmeval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "2bfaae18-d461-4732-876c-fe59e9e49fbe",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!rm -Rf ~/.cache/pip/*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "c3cf4f1e-3cda-46d2-b293-114f61b07168",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from fmeval.data_loaders.data_config import DataConfig\n",
    "from fmeval.model_runners.sm_model_runner import SageMakerModelRunner\n",
    "from fmeval.constants import MIME_TYPE_JSONLINES\n",
    "from fmeval.eval_algorithms.factual_knowledge import FactualKnowledge, FactualKnowledgeConfig\n",
    "from fmeval.eval import get_eval_algorithm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "f100a07c-d076-4dff-aa1a-5670ab9218a7",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sagemaker.config INFO - Not applying SDK defaults from location: /etc/xdg/sagemaker/config.yaml\n",
      "sagemaker.config INFO - Not applying SDK defaults from location: /root/.config/sagemaker/config.yaml\n"
     ]
    }
   ],
   "source": [
    "model = SageMakerModelRunner(\n",
    "    endpoint_name = 'mixtral-8bit-20231213-122707',\n",
    "    content_template =  '{\"inputs\": $prompt, \"parameters\": {\"max_new_tokens\": 50}}',\n",
    "    output=\"[0].generated_text\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "06884a17-b3d8-44ff-bd3f-4bdc340860d2",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Evaluate factual_knowledge\n",
    "eval_algorithm_config = FactualKnowledgeConfig(\"<OR>\")\n",
    "eval_algo = FactualKnowledge(eval_algorithm_config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "59cba5e9-6f17-4478-b362-d77850f8faae",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Custom dataset\n",
    "dataset_config = DataConfig(\n",
    "        dataset_name=\"TREX\",\n",
    "        dataset_uri=\"trex_sample.jsonl\",\n",
    "        dataset_mime_type=\"application/jsonlines\",\n",
    "        model_input_location=\"question\",\n",
    "        target_output_location=\"answers\",\n",
    "        category_location=\"knowledge_category\",\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "ba879f9e-e2b2-4703-81fc-846e598d626d",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-12-13 15:29:12,028\tINFO read_api.py:406 -- To satisfy the requested parallelism of 4, each read task output is split into 4 smaller blocks.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Read progress 0:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Read progress 0:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-12-13 15:29:12,149\tINFO streaming_executor.py:93 -- Executing DAG InputDataBuffer[Input] -> AllToAllOperator[Repartition]\n",
      "2023-12-13 15:29:12,151\tINFO streaming_executor.py:94 -- Execution config: ExecutionOptions(resource_limits=ExecutionResources(cpu=None, gpu=None, object_store_memory=None), locality_with_output=False, preserve_order=True, actor_locality_enabled=True, verbose_progress=False)\n",
      "2023-12-13 15:29:12,152\tINFO streaming_executor.py:96 -- Tip: For detailed progress reporting, run `ray.data.DataContext.get_current().execution_options.verbose_progress = True`\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "- Repartition 1:   0%|          | 0/5 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Split Repartition 2:   0%|          | 0/5 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running 0:   0%|          | 0/5 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-12-13 15:29:13,129\tINFO streaming_executor.py:93 -- Executing DAG InputDataBuffer[Input] -> TaskPoolMapOperator[MapBatches(process_batch)]\n",
      "2023-12-13 15:29:13,131\tINFO streaming_executor.py:94 -- Execution config: ExecutionOptions(resource_limits=ExecutionResources(cpu=None, gpu=None, object_store_memory=None), locality_with_output=False, preserve_order=True, actor_locality_enabled=True, verbose_progress=False)\n",
      "2023-12-13 15:29:13,132\tINFO streaming_executor.py:96 -- Tip: For detailed progress reporting, run `ray.data.DataContext.get_current().execution_options.verbose_progress = True`\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running 0:   0%|          | 0/5 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[2m\u001b[36m(MapBatches(process_batch) pid=2283)\u001b[0m sagemaker.config INFO - Not applying SDK defaults from location: /etc/xdg/sagemaker/config.yaml\n",
      "\u001b[2m\u001b[36m(MapBatches(process_batch) pid=2283)\u001b[0m sagemaker.config INFO - Not applying SDK defaults from location: /root/.config/sagemaker/config.yaml\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-12-13 15:29:14,532\tINFO streaming_executor.py:93 -- Executing DAG InputDataBuffer[Input] -> ActorPoolMapOperator[Map(ModelRunnerWrapper)]\n",
      "2023-12-13 15:29:14,536\tINFO streaming_executor.py:94 -- Execution config: ExecutionOptions(resource_limits=ExecutionResources(cpu=None, gpu=None, object_store_memory=None), locality_with_output=False, preserve_order=True, actor_locality_enabled=True, verbose_progress=False)\n",
      "2023-12-13 15:29:14,538\tINFO streaming_executor.py:96 -- Tip: For detailed progress reporting, run `ray.data.DataContext.get_current().execution_options.verbose_progress = True`\n",
      "2023-12-13 15:29:14,563\tINFO actor_pool_map_operator.py:106 -- Map(ModelRunnerWrapper): Waiting for 1 pool actors to start...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running 0:   0%|          | 0/5 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-12-13 16:29:56,985\tINFO dataset.py:2380 -- Tip: Use `take_batch()` instead of `take() / show()` to return records in pandas or numpy batch format.\n",
      "2023-12-13 16:29:56,988\tINFO streaming_executor.py:93 -- Executing DAG InputDataBuffer[Input] -> TaskPoolMapOperator[Map(_generate_eval_scores)] -> AllToAllOperator[Aggregate] -> LimitOperator[limit=1]\n",
      "2023-12-13 16:29:56,990\tINFO streaming_executor.py:94 -- Execution config: ExecutionOptions(resource_limits=ExecutionResources(cpu=None, gpu=None, object_store_memory=None), locality_with_output=False, preserve_order=True, actor_locality_enabled=True, verbose_progress=False)\n",
      "2023-12-13 16:29:57,001\tINFO streaming_executor.py:96 -- Tip: For detailed progress reporting, run `ray.data.DataContext.get_current().execution_options.verbose_progress = True`\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "- Aggregate 1:   0%|          | 0/5 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Shuffle Map 2:   0%|          | 0/5 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Shuffle Reduce 3:   0%|          | 0/5 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running 0:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[2m\u001b[36m(Map(_generate_eval_scores) pid=2396)\u001b[0m sagemaker.config INFO - Not applying SDK defaults from location: /etc/xdg/sagemaker/config.yaml\u001b[32m [repeated 3x across cluster]\u001b[0m\n",
      "\u001b[2m\u001b[36m(Map(_generate_eval_scores) pid=2396)\u001b[0m sagemaker.config INFO - Not applying SDK defaults from location: /root/.config/sagemaker/config.yaml\u001b[32m [repeated 3x across cluster]\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-12-13 16:30:00,846\tINFO streaming_executor.py:93 -- Executing DAG InputDataBuffer[Input] -> TaskPoolMapOperator[Map(_generate_eval_scores)] -> LimitOperator[limit=1]\n",
      "2023-12-13 16:30:00,848\tINFO streaming_executor.py:94 -- Execution config: ExecutionOptions(resource_limits=ExecutionResources(cpu=None, gpu=None, object_store_memory=None), locality_with_output=False, preserve_order=True, actor_locality_enabled=True, verbose_progress=False)\n",
      "2023-12-13 16:30:00,850\tINFO streaming_executor.py:96 -- Tip: For detailed progress reporting, run `ray.data.DataContext.get_current().execution_options.verbose_progress = True`\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running 0:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-12-13 16:30:01,052\tINFO streaming_executor.py:93 -- Executing DAG InputDataBuffer[Input] -> TaskPoolMapOperator[Map(_generate_eval_scores)] -> AllToAllOperator[Aggregate] -> TaskPoolMapOperator[MapBatches(<lambda>)]\n",
      "2023-12-13 16:30:01,054\tINFO streaming_executor.py:94 -- Execution config: ExecutionOptions(resource_limits=ExecutionResources(cpu=None, gpu=None, object_store_memory=None), locality_with_output=False, preserve_order=True, actor_locality_enabled=True, verbose_progress=False)\n",
      "2023-12-13 16:30:01,055\tINFO streaming_executor.py:96 -- Tip: For detailed progress reporting, run `ray.data.DataContext.get_current().execution_options.verbose_progress = True`\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "- Aggregate 1:   0%|          | 0/5 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Shuffle Map 2:   0%|          | 0/5 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Shuffle Reduce 3:   0%|          | 0/5 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running 0:   0%|          | 0/5 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Sort Sample 0:   0%|          | 0/5 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-12-13 16:30:02,839\tINFO streaming_executor.py:93 -- Executing DAG InputDataBuffer[Input] -> TaskPoolMapOperator[Map(_generate_eval_scores)] -> AllToAllOperator[Aggregate]\n",
      "2023-12-13 16:30:02,843\tINFO streaming_executor.py:94 -- Execution config: ExecutionOptions(resource_limits=ExecutionResources(cpu=None, gpu=None, object_store_memory=None), locality_with_output=False, preserve_order=True, actor_locality_enabled=True, verbose_progress=False)\n",
      "2023-12-13 16:30:02,845\tINFO streaming_executor.py:96 -- Tip: For detailed progress reporting, run `ray.data.DataContext.get_current().execution_options.verbose_progress = True`\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "- Aggregate 1:   0%|          | 0/5 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Shuffle Map 2:   0%|          | 0/5 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Shuffle Reduce 3:   0%|          | 0/5 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running 0:   0%|          | 0/5 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Sort Sample 0:   0%|          | 0/5 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-12-13 16:30:03,741\tINFO streaming_executor.py:93 -- Executing DAG InputDataBuffer[Input] -> TaskPoolMapOperator[Map(_generate_eval_scores)->Map(<lambda>)]\n",
      "2023-12-13 16:30:03,747\tINFO streaming_executor.py:94 -- Execution config: ExecutionOptions(resource_limits=ExecutionResources(cpu=None, gpu=None, object_store_memory=None), locality_with_output=False, preserve_order=True, actor_locality_enabled=True, verbose_progress=False)\n",
      "2023-12-13 16:30:03,753\tINFO streaming_executor.py:96 -- Tip: For detailed progress reporting, run `ray.data.DataContext.get_current().execution_options.verbose_progress = True`\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running 0:   0%|          | 0/5 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-12-13 16:30:04,048\tINFO streaming_executor.py:93 -- Executing DAG InputDataBuffer[Input] -> TaskPoolMapOperator[Map(_generate_eval_scores)->Map(<lambda>)]\n",
      "2023-12-13 16:30:04,051\tINFO streaming_executor.py:94 -- Execution config: ExecutionOptions(resource_limits=ExecutionResources(cpu=None, gpu=None, object_store_memory=None), locality_with_output=False, preserve_order=True, actor_locality_enabled=True, verbose_progress=False)\n",
      "2023-12-13 16:30:04,055\tINFO streaming_executor.py:96 -- Tip: For detailed progress reporting, run `ray.data.DataContext.get_current().execution_options.verbose_progress = True`\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running 0:   0%|          | 0/5 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Evaluate model with fmeval built-in dataset\n",
    "eval_outputs = eval_algo.evaluate(model=model, dataset_config=dataset_config, prompt_template=\"$feature\", save=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "d18f302c-1abc-4bd3-9f19-58d12e185367",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[EvalOutput(eval_name='factual_knowledge', dataset_name='TREX', dataset_scores=[EvalScore(name='factual_knowledge', value=0.7661691542288557)], prompt_template='$feature', category_scores=[CategoryScore(name='Capitals', scores=[EvalScore(name='factual_knowledge', value=0.88)]), CategoryScore(name='Subsidiary', scores=[EvalScore(name='factual_knowledge', value=0.6534653465346535)])], output_path='/tmp/eval_results/factual_knowledge_TREX.jsonl', error=None)]\n"
     ]
    }
   ],
   "source": [
    "# Show Evaluation outputs\n",
    "print(eval_outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "0e9c12ad-b8a4-4bd5-9a7f-7fdd63a630db",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[\n",
      "    {\n",
      "        \"eval_name\": \"factual_knowledge\",\n",
      "        \"dataset_name\": \"TREX\",\n",
      "        \"dataset_scores\": [\n",
      "            {\n",
      "                \"name\": \"factual_knowledge\",\n",
      "                \"value\": 0.7661691542288557\n",
      "            }\n",
      "        ],\n",
      "        \"prompt_template\": \"$feature\",\n",
      "        \"category_scores\": [\n",
      "            {\n",
      "                \"name\": \"Capitals\",\n",
      "                \"scores\": [\n",
      "                    {\n",
      "                        \"name\": \"factual_knowledge\",\n",
      "                        \"value\": 0.88\n",
      "                    }\n",
      "                ]\n",
      "            },\n",
      "            {\n",
      "                \"name\": \"Subsidiary\",\n",
      "                \"scores\": [\n",
      "                    {\n",
      "                        \"name\": \"factual_knowledge\",\n",
      "                        \"value\": 0.6534653465346535\n",
      "                    }\n",
      "                ]\n",
      "            }\n",
      "        ],\n",
      "        \"output_path\": \"/tmp/eval_results/factual_knowledge_TREX.jsonl\",\n",
      "        \"error\": null\n",
      "    }\n",
      "]\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "print(json.dumps(eval_outputs, default=vars, indent=4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "06ef55f3-7344-4646-9c5c-a2b62feb3a6d",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"model_input\": \"Primark is a subsidiary of\", \"model_output\": \"Primark is a subsidiary of Associated British Foods. It is a clothing retailer that sells a wide range of clothing, accessories, footwear, homeware, beauty products, and confectionery. Primark has over 370 stores in 12\", \"target_output\": \"Associated British Foods\", \"category\": \"Subsidiary\", \"prompt\": \"Primark is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Chartered Industries is a subsidiary of\", \"model_output\": \"Chartered Industries is a subsidiary of the Chartered Group of Companies, which is a conglomerate of companies with interests in various sectors of the economy.\\n\\nChartered Industries is a leading manufacturer of high quality packaging materials in Nigeria. The company was established in\", \"target_output\": \"ST Engineering\", \"category\": \"Subsidiary\", \"prompt\": \"Chartered Industries is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 0}]}\n",
      "{\"model_input\": \"Palikir is the capital of\", \"model_output\": \"Palikir is the capital of the Federated States of Micronesia, a country in the Pacific Ocean. It is located on the island of Pohnpei. The city has a population of about 7,000 people.\\n\\nPalikir is\", \"target_output\": \"Micronesian<OR>Micronesia\", \"category\": \"Capitals\", \"prompt\": \"Palikir is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Homs is the capital of\", \"model_output\": \"Homs is the capital of the Homs Governorate in central Syria and the third largest city in Syria after Aleppo and Damascus. It is located 162 kilometres (101 mi) north of Damascus. Located on the Orontes\", \"target_output\": \"Homs Governorate\", \"category\": \"Capitals\", \"prompt\": \"Homs is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Naples is the capital of\", \"model_output\": \"Naples is the capital of the Italian region Campania and the third-largest municipality in Italy after Rome and Milan. In 2015, around 975,260 people lived within the city's administrative limits. The Metropolitan City\", \"target_output\": \"Campania region<OR>Campania\", \"category\": \"Capitals\", \"prompt\": \"Naples is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Naha is the capital of\", \"model_output\": \"Naha is the capital of Okinawa Prefecture, Japan. It is the political, economic, and education center of Okinawa Prefecture. Naha is located in the southern part of the main island of Okinawa. It is the largest\", \"target_output\": \"Okinawa Prefecture<OR>Okinawa\", \"category\": \"Capitals\", \"prompt\": \"Naha is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Legazpi City is the capital of\", \"model_output\": \"Legazpi City is the capital of the province of Albay in the Bicol Region. It is the regional center of the Bicol Region and the commercial, financial, educational, and transportation hub of the province.\\n\\nLegazpi City is known for its beautiful landscape, with\", \"target_output\": \"Albay\", \"category\": \"Capitals\", \"prompt\": \"Legazpi City is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Bandar Seri Begawan is the capital of\", \"model_output\": \"Bandar Seri Begawan is the capital of Brunei, a small country on the island of Borneo. The city is located on the Brunei River and has a population of about 250,000 people. Bandar Seri Begawan is a modern\", \"target_output\": \"Brunei Darussalam<OR>Brunei\", \"category\": \"Capitals\", \"prompt\": \"Bandar Seri Begawan is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Def Jam Recordings is a subsidiary of\", \"model_output\": \"Def Jam Recordings is a subsidiary of Universal Music Group, and was founded in 1984 by Rick Rubin and Russell Simmons. The label was originally created to market and promote hip-hop music, but has since expanded to include a wide variety of genres. Def\", \"target_output\": \"Universal Music Group\", \"category\": \"Subsidiary\", \"prompt\": \"Def Jam Recordings is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"BBC Worldwide is a subsidiary of\", \"model_output\": \"BBC Worldwide is a subsidiary of the BBC, responsible for the commercial exploitation of BBC programmes and other properties, including the BBC\\u2019s global television channels.\\n\\nBBC Worldwide is a commercial company that operates in over 50 countries worldwide. It is a wh\", \"target_output\": \"British Broadcasting Corporation\", \"category\": \"Subsidiary\", \"prompt\": \"BBC Worldwide is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 0}]}\n",
      "{\"model_input\": \"Eurowings is a subsidiary of\", \"model_output\": \"Eurowings is a subsidiary of the German airline Lufthansa. It is a low-cost airline that operates flights to various destinations in Europe and North Africa. Eurowings has its main hub at D\\u00fcsseldorf Airport, but it also operates flights\", \"target_output\": \"Lufthansa Group\", \"category\": \"Subsidiary\", \"prompt\": \"Eurowings is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 0}]}\n",
      "{\"model_input\": \"Kindu is the capital of\", \"model_output\": \"Kindu is the capital of Maniema Province in the Democratic Republic of the Congo. It is located on the Lualaba River, a tributary of the Congo River. The city has a population of approximately 200,000 people.\", \"target_output\": \"Maniema province\", \"category\": \"Capitals\", \"prompt\": \"Kindu is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Bodley Head is a subsidiary of\", \"model_output\": \"Bodley Head is a subsidiary of Random House, which is a subsidiary of Bertelsmann AG, which is a subsidiary of Bertelsmann AG, which is a subsidiary of Bertelsmann AG, which is a subsidiary of Bertelsmann AG\", \"target_output\": \"Random House\", \"category\": \"Subsidiary\", \"prompt\": \"Bodley Head is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"The Selmer Company is a subsidiary of\", \"model_output\": \"The Selmer Company is a subsidiary of the Conn-Selmer, Inc. company, which is the largest manufacturer of band and orchestral instruments in the United States. The Selmer Company was founded in 1885 by Henri Selmer, a woodwind player\", \"target_output\": \"Steinway Musical Instruments\", \"category\": \"Subsidiary\", \"prompt\": \"The Selmer Company is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 0}]}\n",
      "{\"model_input\": \"Bonneville International is a subsidiary of\", \"model_output\": \"Bonneville International is a subsidiary of Deseret Management Corporation, a for-profit arm of The Church of Jesus Christ of Latter-day Saints.\\n\\n## History\\n\\nBonneville International was founded in 1964 by the LDS Church as a\", \"target_output\": \"Deseret Management Corporation\", \"category\": \"Subsidiary\", \"prompt\": \"Bonneville International is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"GE Capital is a subsidiary of\", \"model_output\": \"GE Capital is a subsidiary of General Electric that provides commercial lending and leasing. GE Capital operates in over 50 countries worldwide and is headquartered in Norwalk, Connecticut.\\n\\nGE Capital offers a variety of financial products and services to businesses of all\", \"target_output\": \"General Electric\", \"category\": \"Subsidiary\", \"prompt\": \"GE Capital is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Norcem is a subsidiary of\", \"model_output\": \"Norcem is a subsidiary of HeidelbergCement, one of the world\\u2019s largest building materials companies. Norcem operates two cement plants in Norway, one in Brevik and one in Kj\\u00f8psvik. The Brevik plant is the largest cement\", \"target_output\": \"HeidelbergCement\", \"category\": \"Subsidiary\", \"prompt\": \"Norcem is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Kabul is the capital of\", \"model_output\": \"Kabul is the capital of Afghanistan and the largest city of the country. It is located in the eastern part of the country, at an altitude of 1,791 meters above sea level. The city has a population of about 4.6 million people.\", \"target_output\": \"Islamic State<OR>Afghan<OR>Taliban government<OR>Hindu Shahi<OR>Taliban<OR>Afghanistan\", \"category\": \"Capitals\", \"prompt\": \"Kabul is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Hattusa is the capital of\", \"model_output\": \"Hattusa is the capital of the Hittite Empire in Anatolia, now in central Turkey. It is near the present city of Bo\\u011fazkale, formerly known as Bo\\u011fazk\\u00f6y. The site consists of the ruins of several temples, royal\", \"target_output\": \"Hittites<OR>Hittite<OR>Hittite Empire\", \"category\": \"Capitals\", \"prompt\": \"Hattusa is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Arpad is the capital of\", \"model_output\": \"Arpad is the capital of the Arpadian Empire, and the largest city in the world. It is located in the center of the continent of Arpad, and is the seat of the Arpadian Emperor. The city is home to the Imperial Palace, the Imperial Cat\", \"target_output\": \"Bit Agusi\", \"category\": \"Capitals\", \"prompt\": \"Arpad is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 0}]}\n",
      "{\"model_input\": \"Hasbro Interactive is a subsidiary of\", \"model_output\": \"Hasbro Interactive is a subsidiary of Hasbro, Inc. that was formed in 1995 to develop and publish interactive entertainment products. The company was formed by the merger of Hasbro's Games.com and MicroProse, Inc.\\n\\nHasbro Inter\", \"target_output\": \"Hasbro\", \"category\": \"Subsidiary\", \"prompt\": \"Hasbro Interactive is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Aegon UK is a subsidiary of\", \"model_output\": \"Aegon UK is a subsidiary of Aegon NV, a multinational life insurance, pensions and asset management company headquartered in The Hague, Netherlands. Aegon UK provides pensions, life insurance, asset management and financial advice to individuals and companies in\", \"target_output\": \"Aegon N.V.\", \"category\": \"Subsidiary\", \"prompt\": \"Aegon UK is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 0}]}\n",
      "{\"model_input\": \"Marvel Television is a subsidiary of\", \"model_output\": \"Marvel Television is a subsidiary of Marvel Entertainment, LLC, a wholly owned subsidiary of The Walt Disney Company. Marvel Television develops and produces some of the world\\u2019s most creative and popular live-action and animated programming for broadcast and cable television networks as well as for\", \"target_output\": \"Marvel Entertainment\", \"category\": \"Subsidiary\", \"prompt\": \"Marvel Television is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"San Fernando del Valle de Catamarca is the capital of\", \"model_output\": \"San Fernando del Valle de Catamarca is the capital of the Catamarca Province in Argentina. It is located in the central-western part of the country, in the valley of the same name, surrounded by the Aconquija and Ambato mountain ranges.\\n\\nThe city has a population of\", \"target_output\": \"Catamarca Province<OR>Catamarca\", \"category\": \"Capitals\", \"prompt\": \"San Fernando del Valle de Catamarca is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Edo is the capital of\", \"model_output\": \"Edo is the capital of the Edo State in Nigeria. It is situated in the southern part of Nigeria. The city is located on the north bank of the Benin River. The city is the center of the Edo people. The city is also known as Benin\", \"target_output\": \"Tokugawa shogunate<OR>Tokugawa Japan<OR>Japan<OR>Tokugawa Shogunate\", \"category\": \"Capitals\", \"prompt\": \"Edo is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 0}]}\n",
      "{\"model_input\": \"DB Regio AG is a subsidiary of\", \"model_output\": \"DB Regio AG is a subsidiary of Deutsche Bahn AG and is responsible for regional transport in Germany. The company operates in 13 of the 16 federal states and is the largest provider of regional transport services in Germany. DB Regio AG operates a fleet of around 2\", \"target_output\": \"Deutsche Bahn<OR>Deutsche Bahn AG\", \"category\": \"Subsidiary\", \"prompt\": \"DB Regio AG is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Agartala is the capital of\", \"model_output\": \"Agartala is the capital of the Indian state of Tripura. It is also the second-largest city in North-east India after Guwahati, and is the third-largest urban area in North-east India after Guwahati and Imph\", \"target_output\": \"Tripura\", \"category\": \"Capitals\", \"prompt\": \"Agartala is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Packard Bell is a subsidiary of\", \"model_output\": \"Packard Bell is a subsidiary of Acer, and is a brand that is well known for its budget PCs and laptops. The company has now launched a new range of laptops in India, which includes the EasyNote TS11, EasyNote TS13\", \"target_output\": \"Acer\", \"category\": \"Subsidiary\", \"prompt\": \"Packard Bell is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Mitsubishi Motors Australia Limited is a subsidiary of\", \"model_output\": \"Mitsubishi Motors Australia Limited is a subsidiary of Mitsubishi Motors Corporation, Japan. Mitsubishi Motors Australia Limited (MMAL) was established in 1980 and is headquartered in Adelaide, South Australia.\\n\\nMMAL has a\", \"target_output\": \"Mitsubishi Motors Corporation\", \"category\": \"Subsidiary\", \"prompt\": \"Mitsubishi Motors Australia Limited is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Lucena City is the capital of\", \"model_output\": \"Lucena City is the capital of the province of Quezon, Philippines. It is the largest city in Quezon in terms of population and land area. The city is located at the southernmost tip of the Bondoc Peninsula, the largest peninsula in the Philippines.\\n\\n\", \"target_output\": \"Quezon Province<OR>Quezon\", \"category\": \"Capitals\", \"prompt\": \"Lucena City is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Sukhum is the capital of\", \"model_output\": \"Sukhum is the capital of Abkhazia, a disputed territory on the Black Sea coast. It is a subtropical paradise with palm trees, orange groves and sandy beaches. The city has a rich history dating back to the 6th\", \"target_output\": \"Abkhazia\", \"category\": \"Capitals\", \"prompt\": \"Sukhum is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Binney & Smith is a subsidiary of\", \"model_output\": \"Binney & Smith is a subsidiary of Berol Corporation, which is a subsidiary of Newell Rubbermaid. Binney & Smith is the manufacturer of Crayola crayons, markers, paints, modeling clay, and other art supplies.\\n\\nThe\", \"target_output\": \"Hallmark Cards\", \"category\": \"Subsidiary\", \"prompt\": \"Binney & Smith is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 0}]}\n",
      "{\"model_input\": \"Stroytransgaz is a subsidiary of\", \"model_output\": \"Stroytransgaz is a subsidiary of Gazprom, the Russian gas monopoly. It is building the Nord Stream 2 pipeline, which will carry gas from Russia to Germany.\\n\\nThe company is also building the Turkish Stream pipeline, which will carry gas from Russia to Turkey.\", \"target_output\": \"Gazprom\", \"category\": \"Subsidiary\", \"prompt\": \"Stroytransgaz is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Germanwings is a subsidiary of\", \"model_output\": \"Germanwings is a subsidiary of Lufthansa, the largest German airline. Germanwings is a low-cost airline that operates flights within Europe.\\n\\n## Organization\\n\\nGermanwings is a subsidiary of Lufthansa.\", \"target_output\": \"Lufthansa\", \"category\": \"Subsidiary\", \"prompt\": \"Germanwings is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Baskin-Robbins is a subsidiary of\", \"model_output\": \"Baskin-Robbins is a subsidiary of Dunkin\\u2019 Brands, Inc. and is the world\\u2019s largest chain of ice cream specialty shops. Baskin-Robbins creates and markets innovative, premium ice cream, specialty frozen desserts and beverages,\", \"target_output\": \"Dunkin' Brands\", \"category\": \"Subsidiary\", \"prompt\": \"Baskin-Robbins is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 0}]}\n",
      "{\"model_input\": \"Georgia-Pacific Corporation is a subsidiary of\", \"model_output\": \"Georgia-Pacific Corporation is a subsidiary of Koch Industries, Inc. and is one of the world's leading manufacturers and marketers of building products and related chemicals. The company's products are used extensively in homebuilding, commercial construction, and renovation as well as in\", \"target_output\": \"Koch Industries\", \"category\": \"Subsidiary\", \"prompt\": \"Georgia-Pacific Corporation is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Budapest is the capital of\", \"model_output\": \"Budapest is the capital of Hungary and the country\\u2019s largest city. It is located in the north-central part of the country, on the Danube River. Budapest is a major cultural, economic, and political center of Hungary. The city has a population\", \"target_output\": \"Hungarian<OR>Pest<OR>Hungarians<OR>Hungary<OR>Pest County<OR>Austro-Hungarian Empire\", \"category\": \"Capitals\", \"prompt\": \"Budapest is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Exit Tunes is a subsidiary of\", \"model_output\": \"Exit Tunes is a subsidiary of the Japanese record label Lantis. It was founded in 2007 and is known for producing music for anime, video games, and other media. The label has released music by various artists, including Faylan, GARN\", \"target_output\": \"Pony Canyon\", \"category\": \"Subsidiary\", \"prompt\": \"Exit Tunes is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 0}]}\n",
      "{\"model_input\": \"Seiko Instruments is a subsidiary of\", \"model_output\": \"Seiko Instruments is a subsidiary of Seiko Holdings Corporation, a Japanese company that is well known for its watches and clocks. Seiko Instruments is a global leader in the design and manufacture of electronic components and devices. The company\\u2019s product portfolio includes a wide\", \"target_output\": \"Seiko\", \"category\": \"Subsidiary\", \"prompt\": \"Seiko Instruments is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Hobart is the capital of\", \"model_output\": \"Hobart is the capital of Tasmania, an island state of Australia. It is located on the southeast coast of Tasmania at the mouth of the Derwent River. Hobart is Australia's second-oldest capital city after Sydney, and it is\", \"target_output\": \"Tasmania\", \"category\": \"Capitals\", \"prompt\": \"Hobart is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Digital Telecommunications Philippines is a subsidiary of\", \"model_output\": \"Digital Telecommunications Philippines is a subsidiary of PLDT, the largest telecommunications company in the Philippines. It was established in 1994 as a joint venture between PLDT and Nippon Telegraph and Telephone Corporation (NTT) of Japan. The company\", \"target_output\": \"Philippine Long Distance Telephone Company (PLDT)\", \"category\": \"Subsidiary\", \"prompt\": \"Digital Telecommunications Philippines is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 0}]}\n",
      "{\"model_input\": \"la Seu d'Urgell is the capital of\", \"model_output\": \"la Seu d'Urgell is the capital of the comarca of the Alt Urgell, in the province of Lleida, Catalonia, Spain. It is located in the Pyrenees, at the confluence of the Segre and the Valira rivers. The\", \"target_output\": \"Alt Urgell\", \"category\": \"Capitals\", \"prompt\": \"la Seu d'Urgell is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Madrid is the capital of\", \"model_output\": \"Madrid is the capital of Spain and the largest municipality in both the Community of Madrid and Spain as a whole. The city has a population of almost 3.2 million with a metro area population of approximately 6.5 million. It is the third-larg\", \"target_output\": \"Spanish Republic<OR>Spanish Republican<OR>Spain<OR>Republican<OR>Spanish Empire<OR>Republicans<OR>Spanish\", \"category\": \"Capitals\", \"prompt\": \"Madrid is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"S\\u00e3o Salvador is the capital of\", \"model_output\": \"S\\u00e3o Salvador is the capital of the province of Bahia and the third largest city in Brazil. It is located on the Atlantic coast of the northeastern part of the country. The city was founded in 1549 by the Portuguese and was the first capital of Brazil\", \"target_output\": \"Zaire Province\", \"category\": \"Capitals\", \"prompt\": \"S\\u00e3o Salvador is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 0}]}\n",
      "{\"model_input\": \"Aracaju is the capital of\", \"model_output\": \"Aracaju is the capital of the state of Sergipe in northeastern Brazil. It is located on the Atlantic Ocean, and is the third richest Brazilian state capital in terms of GDP per capita. The city is named after the Aracaju fish\", \"target_output\": \"Sergipe\", \"category\": \"Capitals\", \"prompt\": \"Aracaju is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Port Moresby is the capital of\", \"model_output\": \"Port Moresby is the capital of Papua New Guinea and the largest city in the Pacific Islands outside of Australia and New Zealand. It is located on the southeastern coast of the island of New Guinea. The city has a population of approximately 350,0\", \"target_output\": \"Papua New Guinea<OR>Papua\", \"category\": \"Capitals\", \"prompt\": \"Port Moresby is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Elista is the capital of\", \"model_output\": \"Elista is the capital of the Republic of Kalmykia, Russia. It is located in the southeastern part of the European part of Russia, on the Yergeni Plateau, 240 kilometers (150 mi) south of\", \"target_output\": \"Kalmykia<OR>Kalmyk\", \"category\": \"Capitals\", \"prompt\": \"Elista is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Telsim is a subsidiary of\", \"model_output\": \"Telsim is a subsidiary of Turkcell, the leading mobile operator in Turkey. Telsim is the second largest mobile operator in Turkey with 12.5 million subscribers. Telsim is a GSM 900/1800 MHz\", \"target_output\": \"Vodafone Group\", \"category\": \"Subsidiary\", \"prompt\": \"Telsim is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 0}]}\n",
      "{\"model_input\": \"Scandinavian Airlines is a subsidiary of\", \"model_output\": \"Scandinavian Airlines is a subsidiary of SAS Group, which is a Swedish airline company. It is the flag carrier of Sweden, Norway, and Denmark. The airline was founded in 1946 and is headquartered in Stockholm, Sweden. Scandin\", \"target_output\": \"SAS Group\", \"category\": \"Subsidiary\", \"prompt\": \"Scandinavian Airlines is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Crystal Dynamics is a subsidiary of\", \"model_output\": \"Crystal Dynamics is a subsidiary of Square Enix and is the developer of the Tomb Raider series. The studio is currently working on a new Avengers game, which is set to be released in May 2020.\\n\\n## Is Crystal Dynamics owned\", \"target_output\": \"Square Enix\", \"category\": \"Subsidiary\", \"prompt\": \"Crystal Dynamics is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Gyeongju is the capital of\", \"model_output\": \"Gyeongju is the capital of the Silla Kingdom, which ruled the southern half of the Korean peninsula from the 1st century BC to the 10th century AD. The city is home to a large number of historical sites and monuments, including the Bulgu\", \"target_output\": \"Silla Dynasty<OR>Silla<OR>Silla Kingdom\", \"category\": \"Capitals\", \"prompt\": \"Gyeongju is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Lichinga is the capital of\", \"model_output\": \"Lichinga is the capital of Niassa Province in Mozambique. It is located in the north of the country, near the Malawi border. The city has a population of about 150,000 people.\\n\\nLichinga is a\", \"target_output\": \"Niassa province<OR>Niassa Province\", \"category\": \"Capitals\", \"prompt\": \"Lichinga is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Selmer is a subsidiary of\", \"model_output\": \"Selmer is a subsidiary of the Henri Selmer Paris company, which was founded in 1885 by Henri Selmer. The company is known for its woodwind and brass instruments, including saxophones, clarinets, and trumpets. Selmer instruments\", \"target_output\": \"Steinway Musical Instruments\", \"category\": \"Subsidiary\", \"prompt\": \"Selmer is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 0}]}\n",
      "{\"model_input\": \"News Limited is a subsidiary of\", \"model_output\": \"News Limited is a subsidiary of News Corporation, which is a publicly traded company listed on the NASDAQ stock exchange under the ticker symbol NWS. News Limited is a media company that operates in Australia and New Zealand, and it is one of the largest media companies in these\", \"target_output\": \"News Corporation\", \"category\": \"Subsidiary\", \"prompt\": \"News Limited is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Baghdad is the capital of\", \"model_output\": \"Baghdad is the capital of Iraq and the largest city in the country. It is located in the center of the country on the Tigris River. Baghdad is the political, cultural, and economic center of Iraq. The city has a population of about 7 million\", \"target_output\": \"Abassids<OR>Abbasid<OR>Abbasid caliphate<OR>Abbasid Caliphate<OR>Baghdad Governorate<OR>Iraq\", \"category\": \"Capitals\", \"prompt\": \"Baghdad is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Riga is the capital of\", \"model_output\": \"Riga is the capital of Latvia and the largest city in the Baltic States. It is located on the Gulf of Riga, at the mouth of the Daugava River. Riga is known for its beautiful Art Nouveau architecture, its vibrant cultural scene\", \"target_output\": \"Latvia<OR>Latvian\", \"category\": \"Capitals\", \"prompt\": \"Riga is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Newsquest media group is a subsidiary of\", \"model_output\": \"Newsquest media group is a subsidiary of Gannett Company, Inc. and is one of the largest regional news publishers in the UK. It publishes more than 150 newspapers and associated websites, including the Northern Echo, the Herald, the Argus and\", \"target_output\": \"Gannett\", \"category\": \"Subsidiary\", \"prompt\": \"Newsquest media group is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"MDA Space Missions is a subsidiary of\", \"model_output\": \"MDA Space Missions is a subsidiary of MDA, a Maxar Technologies company. MDA Space Missions is a leading provider of space robotics, satellite systems and geointelligence solutions.\\n\\nMDA Space Missions is a leading provider of space robotics, satellite systems\", \"target_output\": \"MacDonald Dettwiler\", \"category\": \"Subsidiary\", \"prompt\": \"MDA Space Missions is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 0}]}\n",
      "{\"model_input\": \"Crayola is a subsidiary of\", \"model_output\": \"Crayola is a subsidiary of Hallmark Cards, Inc. and is known for its crayons, markers, colored pencils, paints, modeling clay, and other art supplies. The company was founded in 1885 by Edwin Binney\", \"target_output\": \"Hallmark Cards\", \"category\": \"Subsidiary\", \"prompt\": \"Crayola is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Guiyang is the capital of\", \"model_output\": \"Guiyang is the capital of Guizhou Province in Southwest China. It is a city with a long history and rich culture. Guiyang is also a city with a pleasant climate and beautiful scenery. It is a city that is worth visiting.\\n\\n\", \"target_output\": \"Guizhou Province<OR>Guizhou\", \"category\": \"Capitals\", \"prompt\": \"Guiyang is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Surabaya is the capital of\", \"model_output\": \"Surabaya is the capital of East Java province and the second largest city in Indonesia. It is located on the northeastern coast of Java Island, about 750 kilometers east of Jakarta. Surabaya is a major industrial, commercial, and transportation hub in\", \"target_output\": \"Jawa Timur<OR>East Java\", \"category\": \"Capitals\", \"prompt\": \"Surabaya is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Vienna is the capital of\", \"model_output\": \"Vienna is the capital of Austria and one of the nine states of Austria. Vienna is Austria's primary city, with a population of about 1.7 million (2.4 million within the metropolitan area, more than 20% of Austria's population\", \"target_output\": \"Austro\\u2013Hungarian<OR>Austro-Hungarian Empire<OR>Holy Roman Empire<OR>Austro-Hungarian<OR>Lower Austria<OR>Austria<OR>Austria-Hungarian\", \"category\": \"Capitals\", \"prompt\": \"Vienna is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Paramount Home Media Distribution is a subsidiary of\", \"model_output\": \"Paramount Home Media Distribution is a subsidiary of Paramount Pictures Corporation, a global producer and distributor of filmed entertainment.\\n\\nParamount Home Media Distribution (PHMD) is responsible for the sales, marketing and distribution of home entertainment products on behalf of Paramount Pictures, Paramount An\", \"target_output\": \"Paramount Pictures\", \"category\": \"Subsidiary\", \"prompt\": \"Paramount Home Media Distribution is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Bassein is the capital of\", \"model_output\": \"Bassein is the capital of the Bassein District in the Irrawaddy Division of Burma. It is located on the Pathein River, 190 kilometers west of Yangon. The city has a population of 130,000\", \"target_output\": \"Ayeyarwady Region\", \"category\": \"Capitals\", \"prompt\": \"Bassein is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 0}]}\n",
      "{\"model_input\": \"Fox Entertainment Group is a subsidiary of\", \"model_output\": \"Fox Entertainment Group is a subsidiary of News Corporation, and is the parent company of the Fox Broadcasting Company, Fox Television Stations, Fox Cable Networks, Fox Sports Net, Fox News Channel, Fox Movie Channel, Fox Television Studios, and 20th Century Fox\", \"target_output\": \"21st Century Fox\", \"category\": \"Subsidiary\", \"prompt\": \"Fox Entertainment Group is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 0}]}\n",
      "{\"model_input\": \"East Berlin is the capital of\", \"model_output\": \"East Berlin is the capital of the German Democratic Republic (GDR, or East Germany). It is located in the Soviet sector of post-World War II Berlin. The city is divided from West Berlin by the Berlin Wall, which is built in 1961. East\", \"target_output\": \"German Democratic Republic<OR>German Democratic Republic (GDR)<OR>East Germany\", \"category\": \"Capitals\", \"prompt\": \"East Berlin is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Pul-i-Alam is the capital of\", \"model_output\": \"Pul-i-Alam is the capital of Logar Province, Afghanistan, and is located on the main Kabul-Kandahar highway about 70 km south of Kabul. The city has a population of about 100,000 people.\\n\\nP\", \"target_output\": \"Logar Province\", \"category\": \"Capitals\", \"prompt\": \"Pul-i-Alam is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Buriram is the capital of\", \"model_output\": \"Buriram is the capital of Buriram Province in Thailand. It is a small city with a population of about 28,000 people. The city is located in the northeastern part of Thailand, about 400 kilometers from Bangkok\", \"target_output\": \"Buriram Province<OR>Buri Ram\", \"category\": \"Capitals\", \"prompt\": \"Buriram is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Ulan-Ude is the capital of\", \"model_output\": \"Ulan-Ude is the capital of the Republic of Buryatia, Russia. It is located in the southeastern part of Siberia, on the banks of the Uda River, about 100 kilometers (62 miles) southeast of Lake Ba\", \"target_output\": \"Buryat<OR>Buryatia\", \"category\": \"Capitals\", \"prompt\": \"Ulan-Ude is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"2K Czech is a subsidiary of\", \"model_output\": \"2K Czech is a subsidiary of 2K Games, and is the developer of the Mafia series.\\n\\n## Latest Games\\n\\n- ### Mafia II (2010)\\n\\nMafia II is an open world, third-person\", \"target_output\": \"2K Games\", \"category\": \"Subsidiary\", \"prompt\": \"2K Czech is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Rolls-Royce Deutschland is a subsidiary of\", \"model_output\": \"Rolls-Royce Deutschland is a subsidiary of Rolls-Royce plc, a global power systems company. The company is headquartered in Dahlewitz, near Berlin, Germany.\\n\\nRolls-Royce Deutschland is responsible for the design, development\", \"target_output\": \"Rolls-Royce plc\", \"category\": \"Subsidiary\", \"prompt\": \"Rolls-Royce Deutschland is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"MD Robotics is a subsidiary of\", \"model_output\": \"MD Robotics is a subsidiary of MacDonald, Dettwiler and Associates Ltd. (MDA), a global communications and information company providing technology solutions to commercial and government organizations worldwide.\\n\\nMD Robotics is a world leader in the design, development, manufacture\", \"target_output\": \"MacDonald Dettwiler\", \"category\": \"Subsidiary\", \"prompt\": \"MD Robotics is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 0}]}\n",
      "{\"model_input\": \"Sky UK Limited is a subsidiary of\", \"model_output\": \"Sky UK Limited is a subsidiary of Comcast, providing television and broadband Internet services, fixed line and mobile telephone services to consumers and businesses in the United Kingdom. It was formed in 1990 by the merger of Sky Television and British Satellite Broadcasting\", \"target_output\": \"Sky plc\", \"category\": \"Subsidiary\", \"prompt\": \"Sky UK Limited is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 0}]}\n",
      "{\"model_input\": \"Albany is the capital of\", \"model_output\": \"Albany is the capital of New York State and the county seat of Albany County. It is located on the west bank of the Hudson River approximately 10 miles (16 km) south of its confluence with the Mohawk River and approximately 135\", \"target_output\": \"Albany County<OR>New York State<OR>New York\", \"category\": \"Capitals\", \"prompt\": \"Albany is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Manama is the capital of\", \"model_output\": \"Manama is the capital of Bahrain, a small island country in the Persian Gulf. The city is located on the northeastern coast of Bahrain Island, the largest island in the country. Manama is the country\\u2019s largest city and its political,\", \"target_output\": \"Bahrain\", \"category\": \"Capitals\", \"prompt\": \"Manama is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Nelspruit is the capital of\", \"model_output\": \"Nelspruit is the capital of the Mpumalanga province in South Africa. It is a small city with a population of about 200,000 people. Nelspruit is a popular tourist destination because it is close to the Kruger National Park\", \"target_output\": \"Mpumalanga\", \"category\": \"Capitals\", \"prompt\": \"Nelspruit is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"RTL Group is a subsidiary of\", \"model_output\": \"RTL Group is a subsidiary of Bertelsmann, a German multinational media corporation. RTL Group is a European entertainment network, with interests in 60 television channels and 31 radio stations in Germany, France, the Benelux countries, Spain and Central and\", \"target_output\": \"Bertelsmann\", \"category\": \"Subsidiary\", \"prompt\": \"RTL Group is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Port au Prince is the capital of\", \"model_output\": \"Port au Prince is the capital of Haiti, a country in the Caribbean. It is the largest city in Haiti and the economic center of the country. The city has a population of about 2.6 million people.\\n\\nPort au Prince is located on the\", \"target_output\": \"Haiti\", \"category\": \"Capitals\", \"prompt\": \"Port au Prince is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Nokia Networks is a subsidiary of\", \"model_output\": \"Nokia Networks is a subsidiary of Nokia Corporation, which provides telecommunications infrastructure and services. Nokia Networks is headquartered in Espoo, Finland, and has operations in over 120 countries. The company provides a range of products and services\", \"target_output\": \"Nokia Corporation<OR>Nokia\", \"category\": \"Subsidiary\", \"prompt\": \"Nokia Networks is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Buda is the capital of\", \"model_output\": \"Buda is the capital of Pest County, Hungary. It is the western part of the capital city of Budapest, and on the west bank of the Danube. Buda is built on the right side of the river, and Pest on the left side. The\", \"target_output\": \"Hungary\", \"category\": \"Capitals\", \"prompt\": \"Buda is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Makassar is the capital of\", \"model_output\": \"Makassar is the capital of South Sulawesi province in Indonesia. It is the largest city on Sulawesi Island and the fifth largest city in Indonesia. Makassar is located on the southwest coast of Sulawesi, on the Makassar Strait, and\", \"target_output\": \"South Sulawesi\", \"category\": \"Capitals\", \"prompt\": \"Makassar is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Gelnhausen is the capital of\", \"model_output\": \"Gelnhausen is the capital of the Main-Kinzig district in the German state of Hesse. It is located in the Kinzig valley, about 50 km east of Frankfurt am Main. The town is known for its well-preserved medieval old town and\", \"target_output\": \"Main-Kinzig-Kreis\", \"category\": \"Capitals\", \"prompt\": \"Gelnhausen is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 0}]}\n",
      "{\"model_input\": \"Amsterdam is the capital of\", \"model_output\": \"Amsterdam is the capital of the Netherlands and is one of the most popular tourist destinations in Europe. The city is known for its beautiful canals, historic buildings, and vibrant nightlife.\\n\\nOne of the most iconic sights in Amsterdam is the network of\", \"target_output\": \"Dutch<OR>The Netherlands<OR>Netherlands\", \"category\": \"Capitals\", \"prompt\": \"Amsterdam is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Assurances G\\u00e9n\\u00e9rales de France is a subsidiary of\", \"model_output\": \"Assurances G\\u00e9n\\u00e9rales de France is a subsidiary of the AXA Group, the world's largest insurance company. AGF is the second largest insurance company in France, and is also active in the United Kingdom, Spain, Italy, Germany, Belgium, the Netherlands, and the United States.\", \"target_output\": \"Allianz SE\", \"category\": \"Subsidiary\", \"prompt\": \"Assurances G\\u00e9n\\u00e9rales de France is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 0}]}\n",
      "{\"model_input\": \"HSBC Bank plc is a subsidiary of\", \"model_output\": \"HSBC Bank plc is a subsidiary of HSBC Holdings plc, the largest banking and financial services organisation in the world. HSBC Bank plc is the largest bank in the UK and the largest banking group in Europe.\\n\\nHSBC Bank plc is a universal\", \"target_output\": \"HSBC Holdings\", \"category\": \"Subsidiary\", \"prompt\": \"HSBC Bank plc is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"SouthernLINC Wireless is a subsidiary of\", \"model_output\": \"SouthernLINC Wireless is a subsidiary of Southern Company, a leading energy company in the United States. SouthernLINC Wireless provides wireless communications services to businesses and consumers in the southeastern United States, including Alabama, Georgia, Florida, Mississippi, and Tennessee.\\n\\nSouthern\", \"target_output\": \"Southern Company\", \"category\": \"Subsidiary\", \"prompt\": \"SouthernLINC Wireless is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Danger Close Games is a subsidiary of\", \"model_output\": \"Danger Close Games is a subsidiary of Electronic Arts, founded in 2009. The studio is based in Los Angeles, California.\\n\\n## Latest Games\\n\\n- ### Medal of Honor: Warfighter (2012)\\n\\nMedal\", \"target_output\": \"DreamWorks SKG\", \"category\": \"Subsidiary\", \"prompt\": \"Danger Close Games is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 0}]}\n",
      "{\"model_input\": \"Bern is the capital of\", \"model_output\": \"Bern is the capital of Switzerland and is located in the German-speaking part of the country. The city is situated on the Aare River and is surrounded by the Swiss Alps. Bern is known for its medieval architecture, including the Zytglogge, a\", \"target_output\": \"Switzerland<OR>Swiss\", \"category\": \"Capitals\", \"prompt\": \"Bern is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Rouen is the capital of\", \"model_output\": \"Rouen is the capital of the region of Normandy in France. It is located on the River Seine, about 90 miles from Paris. Rouen is known for its beautiful architecture, including the Rouen Cathedral, which was painted by Claude Monet. The city\", \"target_output\": \"Normandy<OR>Normandie<OR>Norman\", \"category\": \"Capitals\", \"prompt\": \"Rouen is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Minsk is the capital of\", \"model_output\": \"Minsk is the capital of Belarus and its largest city. It is located in the center of the country on the Svislach and Nyamiha rivers. The city has a population of about 2 million people.\\n\\nMinsk is an important cultural\", \"target_output\": \"Byelorussian Soviet Socialist Republic<OR>Belarus<OR>Belarusian<OR>Minsk Region\", \"category\": \"Capitals\", \"prompt\": \"Minsk is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Bloemfontein is the capital of\", \"model_output\": \"Bloemfontein is the capital of the Free State Province of South Africa. It is the sixth largest city in South Africa and is the judicial capital of the nation, one of South Africa's three national capitals (the other two being Pretoria, the executive capital,\", \"target_output\": \"Free State<OR>Mangaung Local Municipality<OR>Orange Free State<OR>Free State Province<OR>South Africa\", \"category\": \"Capitals\", \"prompt\": \"Bloemfontein is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Kenworth is a subsidiary of\", \"model_output\": \"Kenworth is a subsidiary of PACCAR, which is a leading global technology company that designs, manufactures and customer supports high-quality light-, medium- and heavy-duty trucks under the Kenworth, Peterbilt and DAF nameplates. PACCAR\", \"target_output\": \"Paccar\", \"category\": \"Subsidiary\", \"prompt\": \"Kenworth is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Tarin Kowt is the capital of\", \"model_output\": \"Tarin Kowt is the capital of Uruzgan Province in Afghanistan. It is located in the central part of the province, in the valley of the Helmand River. The city has a population of about 20,000 people.\\n\\nTarin Kow\", \"target_output\": \"Urozgan Province<OR>Uruzgan province<OR>Uruzgan\", \"category\": \"Capitals\", \"prompt\": \"Tarin Kowt is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Digne-les-Bains is the capital of\", \"model_output\": \"Digne-les-Bains is the capital of the Alpes-de-Haute-Provence department in the Provence-Alpes-C\\u00f4te d'Azur region of southeastern France.\\n\\n## Get in\\n\\n### By car\\n\\nDigne-les\", \"target_output\": \"Alpes-de-Haute-Provence\", \"category\": \"Capitals\", \"prompt\": \"Digne-les-Bains is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Granada Sky Broadcasting is a subsidiary of\", \"model_output\": \"Granada Sky Broadcasting is a subsidiary of ITV plc, which owns the Granada Television and London Weekend Television franchises. It was formed in 1996 to launch a digital television service on the Astra 1B satellite.\\n\\n## History\\n\\nGr\", \"target_output\": \"ITV plc\", \"category\": \"Subsidiary\", \"prompt\": \"Granada Sky Broadcasting is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Atco Records is a subsidiary of\", \"model_output\": \"Atco Records is a subsidiary of Atlantic Records, founded in 1955 by Herb Abramson.\\n\\n## History\\n\\nAtco was founded in 1955 by Herb Abramson, who had been a co-founder\", \"target_output\": \"Warner Music Group\", \"category\": \"Subsidiary\", \"prompt\": \"Atco Records is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 0}]}\n",
      "{\"model_input\": \"Audi is a subsidiary of\", \"model_output\": \"Audi is a subsidiary of the German automobile manufacturer Volkswagen AG and designs, engineers, produces, markets and distributes luxury vehicles. Audi oversees worldwide operations from its headquarters in Ingolstadt, Bavaria, Germany. Audi-branded vehicles are\", \"target_output\": \"Volkswagen Group\", \"category\": \"Subsidiary\", \"prompt\": \"Audi is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 0}]}\n",
      "{\"model_input\": \"Istithmar is a subsidiary of\", \"model_output\": \"Istithmar is a subsidiary of Dubai World, a global investment company with a portfolio of assets in more than 100 cities around the world. Istithmar World is a global investment company with a mandate to seek out and create opportunities in various sectors around the world\", \"target_output\": \"Dubai World\", \"category\": \"Subsidiary\", \"prompt\": \"Istithmar is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Zagreb is the capital of\", \"model_output\": \"Zagreb is the capital of Croatia and the largest city in the country. It is located in the northwest of the country, along the Sava River, and is the political, economic, and cultural center of Croatia. Zagreb is known for its beautiful\", \"target_output\": \"Croatian<OR>Zagreb County<OR>Croatia\", \"category\": \"Capitals\", \"prompt\": \"Zagreb is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Thimphu is the capital of\", \"model_output\": \"Thimphu is the capital of Bhutan and the largest city in the country. It is located in the western central part of Bhutan and is the center of government, religion and commerce. Thimphu is a unique city with an unusual mixture of modern development alongside\", \"target_output\": \"Thimphu District<OR>Bhutan\", \"category\": \"Capitals\", \"prompt\": \"Thimphu is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Marvel Animation is a subsidiary of\", \"model_output\": \"Marvel Animation is a subsidiary of Marvel Entertainment, LLC, a wholly owned subsidiary of The Walt Disney Company. Marvel Animation is responsible for producing and distributing animated television series, direct-to-video films, and other animated media based on Marvel Comics characters and\", \"target_output\": \"Marvel Television\", \"category\": \"Subsidiary\", \"prompt\": \"Marvel Animation is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 0}]}\n",
      "{\"model_input\": \"GOG.com is a subsidiary of\", \"model_output\": \"GOG.com is a subsidiary of CD Projekt, the Polish company behind the Witcher series. It's a digital distribution platform that specializes in DRM-free games, and it's been around since 2008.\\n\\nGOG.com\", \"target_output\": \"CD Projekt\", \"category\": \"Subsidiary\", \"prompt\": \"GOG.com is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Vodafone Italia is a subsidiary of\", \"model_output\": \"Vodafone Italia is a subsidiary of Vodafone Group, the world's largest mobile telecommunications company, and is the second largest mobile operator in Italy.\\n\\nVodafone Italia was formed in 1995 as Omnitel Pronto Italia\", \"target_output\": \"Vodafone Group plc\", \"category\": \"Subsidiary\", \"prompt\": \"Vodafone Italia is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 0}]}\n",
      "{\"model_input\": \"Bamiyan city is the capital of\", \"model_output\": \"Bamiyan city is the capital of Bamiyan province in central Afghanistan. It is located in the Hindu Kush mountain range, at an altitude of about 2,500 meters (8,200 feet). The city has a population of approximately 60\", \"target_output\": \"Bamiyan Province\", \"category\": \"Capitals\", \"prompt\": \"Bamiyan city is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Everything Everywhere is a subsidiary of\", \"model_output\": \"Everything Everywhere is a subsidiary of the British telecommunications company EE Limited, which is a joint venture between Deutsche Telekom and Orange S.A. The company was formed in 2010 as a result of the merger of T-Mobile UK and Orange\", \"target_output\": \"BT Group\", \"category\": \"Subsidiary\", \"prompt\": \"Everything Everywhere is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 0}]}\n",
      "{\"model_input\": \"Frigidaire is a subsidiary of\", \"model_output\": \"Frigidaire is a subsidiary of Electrolux, a Swedish company that is one of the largest appliance manufacturers in the world. Frigidaire is known for its high-quality appliances, including refrigerators, freezers, dishwashers, washing machines,\", \"target_output\": \"Electrolux\", \"category\": \"Subsidiary\", \"prompt\": \"Frigidaire is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Podgorica is the capital of\", \"model_output\": \"Podgorica is the capital of Montenegro, a small country in the Balkans. It is the largest city in Montenegro, with a population of around 150,000 people. Podgorica is located in the central part of\", \"target_output\": \"Montenegro<OR>Montenegrin\", \"category\": \"Capitals\", \"prompt\": \"Podgorica is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"King Edward Point is the capital of\", \"model_output\": \"King Edward Point is the capital of the British Antarctic Territory, and is located on the northern coast of South Georgia Island. The station was established in 1909 as a whaling station, and was taken over by the British government in 1925\", \"target_output\": \"South Georgia<OR>South Sandwich Islands\", \"category\": \"Capitals\", \"prompt\": \"King Edward Point is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"NutraSweet is a subsidiary of\", \"model_output\": \"NutraSweet is a subsidiary of the Monsanto Company, which is the largest producer of genetically engineered seeds in the world. Monsanto is also the company that brought us Agent Orange, DDT, PCBs, and bovine growth hormone.\\n\\n\", \"target_output\": \"Monsanto\", \"category\": \"Subsidiary\", \"prompt\": \"NutraSweet is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Sony Music Japan is a subsidiary of\", \"model_output\": \"Sony Music Japan is a subsidiary of Sony Music Entertainment Japan, which is a subsidiary of Sony Corporation of America. Sony Music Japan is the largest music company in Japan and the second largest in the world. It was founded in 1968 as CBS Sony Records and was\", \"target_output\": \"Sony Corporation\", \"category\": \"Subsidiary\", \"prompt\": \"Sony Music Japan is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Aix-en-Provence is the capital of\", \"model_output\": \"Aix-en-Provence is the capital of the Provence-Alpes-C\\u00f4te d'Azur region in southern France. It is a city of art and culture, with a rich history and a vibrant contemporary scene. Aix-en-Provence is known for its\", \"target_output\": \"Provence\", \"category\": \"Capitals\", \"prompt\": \"Aix-en-Provence is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Constantinople is the capital of\", \"model_output\": \"Constantinople is the capital of the Byzantine Empire. It is the largest and wealthiest city in the world, and is the center of trade between Europe and Asia. It is located on the European side of the Bosphorus Strait, which connects the Black Sea\", \"target_output\": \"Roman Empire<OR>Byzantine Empire<OR>Ottoman<OR>Byzantine/Eastern Roman Empire<OR>Eastern Roman<OR>Byzantine emperors<OR>Turkey<OR>East Roman Emperor<OR>Eastern Roman (Byzantine) Empire<OR>Byzantine<OR>Byzantine emperor<OR>Eastern Roman Empire<OR>Turkish<OR>East Roman<OR>Ottoman Empire<OR>Latin Empire\", \"category\": \"Capitals\", \"prompt\": \"Constantinople is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Yamoussoukro is the capital of\", \"model_output\": \"Yamoussoukro is the capital of C\\u00f4te d'Ivoire (Ivory Coast). It is located in the central part of the country, 240 kilometres (150 mi) north-west of the country's largest city and former capital,\", \"target_output\": \"Ivory Coast<OR>C\\u00f4te d'Ivoire\", \"category\": \"Capitals\", \"prompt\": \"Yamoussoukro is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Zipcar is a subsidiary of\", \"model_output\": \"Zipcar is a subsidiary of Avis Budget Group, which is a leading global provider of vehicle rental services. Zipcar is the world\\u2019s leading car-sharing network, which is available in approximately 500 cities and towns across Belgium, Canada, Costa\", \"target_output\": \"Avis Budget Group\", \"category\": \"Subsidiary\", \"prompt\": \"Zipcar is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Tibotec is a subsidiary of\", \"model_output\": \"Tibotec is a subsidiary of Johnson & Johnson, and is a pharmaceutical company that specializes in the development of antiretroviral drugs for the treatment of HIV. Tibotec was founded in 1997 and is headquartered in Meche\", \"target_output\": \"Johnson & Johnson\", \"category\": \"Subsidiary\", \"prompt\": \"Tibotec is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Warangal is the capital of\", \"model_output\": \"Warangal is the capital of the Kakatiya Empire, which ruled the Telugu region from 1163 to 1323. The Kakatiyas built many temples, forts, lakes, and reservoirs in and around War\", \"target_output\": \"Kakatiya dynasty\", \"category\": \"Capitals\", \"prompt\": \"Warangal is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 0}]}\n",
      "{\"model_input\": \"Chandigarh is the capital of\", \"model_output\": \"Chandigarh is the capital of the northern Indian states of Punjab and Haryana, but the city is not part of either state. The city of Chandigarh was the first planned city in India post-independence in 1947 and is known\", \"target_output\": \"Haryana<OR>Punjab, India<OR>Indian Punjab<OR>Harayana\", \"category\": \"Capitals\", \"prompt\": \"Chandigarh is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Limoges is the capital of\", \"model_output\": \"Limoges is the capital of the Haute-Vienne department in the Limousin region in west-central France. It is a city of art and history, and a major cultural and educational centre.\\n\\nLimoges is famous for its production of porcel\", \"target_output\": \"Limousin<OR>Haute-Vienne\", \"category\": \"Capitals\", \"prompt\": \"Limoges is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Aurillac is the capital of\", \"model_output\": \"Aurillac is the capital of the Cantal department in the Auvergne region of France. It is a small town with a population of around 30,000 people. Aurillac is located in the Massif Central, a large mountain range in the\", \"target_output\": \"Cantal\", \"category\": \"Capitals\", \"prompt\": \"Aurillac is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Marvel Productions is a subsidiary of\", \"model_output\": \"Marvel Productions is a subsidiary of Marvel Entertainment, which is a subsidiary of The Walt Disney Company. Marvel Productions was originally known as DePatie-Freleng Enterprises, which was founded in 1963 by Friz Freleng and David H\", \"target_output\": \"Marvel Entertainment Group\", \"category\": \"Subsidiary\", \"prompt\": \"Marvel Productions is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 0}]}\n",
      "{\"model_input\": \"Hongkong Bank is a subsidiary of\", \"model_output\": \"Hongkong Bank is a subsidiary of HSBC Holdings plc, which is one of the largest banking and financial services organizations in the world. Hongkong Bank provides a wide range of financial services to personal and corporate customers in Hong Kong and mainland China.\\n\\nH\", \"target_output\": \"HSBC Holdings plc\", \"category\": \"Subsidiary\", \"prompt\": \"Hongkong Bank is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"PopCap Games is a subsidiary of\", \"model_output\": \"PopCap Games is a subsidiary of Electronic Arts, and is a developer and publisher of casual video games. PopCap Games was founded in 2000 by John Vechey, Brian Fiete, and Jason Kapalka. The company is headquartered\", \"target_output\": \"Electronic Arts\", \"category\": \"Subsidiary\", \"prompt\": \"PopCap Games is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Mogadishu is the capital of\", \"model_output\": \"Mogadishu is the capital of Somalia and the largest city in the country. It is located on the coast of the Indian Ocean, in the Benadir region. Mogadishu has a population of approximately 2.4 million people, making it the most pop\", \"target_output\": \"Somalia<OR>Banaadir<OR>Somali\", \"category\": \"Capitals\", \"prompt\": \"Mogadishu is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Korea Hydro & Nuclear Power is a subsidiary of\", \"model_output\": \"Korea Hydro & Nuclear Power is a subsidiary of Korea Electric Power Corporation (KEPCO). KHNP is responsible for the operation of 23 nuclear power plants in Korea, which generate about 30% of the country's electricity.\\n\\nKHNP is also involved in\", \"target_output\": \"Korea Electric Power Corporation\", \"category\": \"Subsidiary\", \"prompt\": \"Korea Hydro & Nuclear Power is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Toei Animation is a subsidiary of\", \"model_output\": \"Toei Animation is a subsidiary of Toei Company, Ltd. that specializes in animation. It is the oldest animation studio in Japan and has produced many well-known series, such as Dragon Ball, One Piece, and Sailor Moon.\\n\\nToei Animation\", \"target_output\": \"Toei Company\", \"category\": \"Subsidiary\", \"prompt\": \"Toei Animation is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Google China is a subsidiary of\", \"model_output\": \"Google China is a subsidiary of Google, Inc. that provides search, local, and communication services in the People's Republic of China. Google China was founded in 2005, and its headquarters are in Beijing.\\n\\nGoogle China offers a number of services,\", \"target_output\": \"Google\", \"category\": \"Subsidiary\", \"prompt\": \"Google China is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Sarajevo is the capital of\", \"model_output\": \"Sarajevo is the capital of Bosnia and Herzegovina, a country in southeastern Europe. The city is located in a valley surrounded by mountains. It has a population of about 400,000 people.\\n\\nSarajevo\", \"target_output\": \"Bosnian<OR>Sarajevo Canton<OR>Republika Srpska<OR>Bosnia\", \"category\": \"Capitals\", \"prompt\": \"Sarajevo is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Pedigree Petfoods is a subsidiary of\", \"model_output\": \"Pedigree Petfoods is a subsidiary of Mars, Incorporated, a family-owned business with more than a century of history making diverse products and offering services for people and the pets people love. The Pedigree brand is built on an unwavering love for dogs and\", \"target_output\": \"Mars, Incorporated\", \"category\": \"Subsidiary\", \"prompt\": \"Pedigree Petfoods is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Sitten is the capital of\", \"model_output\": \"Sitten is the capital of the canton of Valais in Switzerland. It is located in the German-speaking part of the canton. The city is situated on the right bank of the Rhone River, at the foot of the western Alps. Sitten is\", \"target_output\": \"Sion<OR>Valais\", \"category\": \"Capitals\", \"prompt\": \"Sitten is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Georgia Power is a subsidiary of\", \"model_output\": \"Georgia Power is a subsidiary of Southern Company, one of the largest producers of electricity in the United States. Georgia Power serves more than 2.5 million customers in all but four of Georgia's 159 counties.\\n\\nGeorgia Power is a reg\", \"target_output\": \"Southern Company\", \"category\": \"Subsidiary\", \"prompt\": \"Georgia Power is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Anheuser-Busch is a subsidiary of\", \"model_output\": \"Anheuser-Busch is a subsidiary of Anheuser-Busch InBev, the world\\u2019s largest brewer. The company\\u2019s history dates back to 1852 when Eberhard Anheuser purchased a small brewery in St. Louis, Missouri.\", \"target_output\": \"Anheuser-Busch InBev\", \"category\": \"Subsidiary\", \"prompt\": \"Anheuser-Busch is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Alor Setar is the capital of\", \"model_output\": \"Alor Setar is the capital of the state of Kedah, Malaysia. It is the biggest city in Kedah and the 13th largest city in Malaysia. Alor Setar is located near the border of Thailand and is the northernmost city in Peninsular\", \"target_output\": \"Kedah\", \"category\": \"Capitals\", \"prompt\": \"Alor Setar is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Senftenberg is the capital of\", \"model_output\": \"Senftenberg is the capital of the Oberspreewald-Lausitz district in the state of Brandenburg, Germany. It is situated on the Black Elster river, 15 km south of Cottbus and 100 km south-southeast\", \"target_output\": \"Oberspreewald-Lausitz\", \"category\": \"Capitals\", \"prompt\": \"Senftenberg is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Kuala Lumpur is the capital of\", \"model_output\": \"Kuala Lumpur is the capital of Malaysia and is a city that is full of life and culture. It is a city that is full of contrasts, with modern skyscrapers standing alongside traditional markets and colonial architecture. Kuala Lumpur is a city that is full\", \"target_output\": \"Malaysia<OR>Malaya<OR>Federated Malay States\", \"category\": \"Capitals\", \"prompt\": \"Kuala Lumpur is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Castries is the capital of\", \"model_output\": \"Castries is the capital of Saint Lucia, a small island nation in the Caribbean. The city is located on the northwest coast of the island and is the commercial center of the country. Castries has a population of about 20,000 people and is\", \"target_output\": \"Saint Lucia<OR>St Lucia\", \"category\": \"Capitals\", \"prompt\": \"Castries is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Activision is a subsidiary of\", \"model_output\": \"Activision is a subsidiary of Activision Blizzard, which is a company that is known for developing and publishing video games. The company was founded in 1979 and is headquartered in Santa Monica, California. Activision is one of the largest\", \"target_output\": \"Activision Blizzard\", \"category\": \"Subsidiary\", \"prompt\": \"Activision is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"BellSouth Corporation is a subsidiary of\", \"model_output\": \"BellSouth Corporation is a subsidiary of AT&T Inc. that provides local telephone services to 24 states in the United States. BellSouth is headquartered in Atlanta, Georgia.\\n\\n### BellSouth problems in the last 24 hours in Covington,\", \"target_output\": \"AT&T Inc.\", \"category\": \"Subsidiary\", \"prompt\": \"BellSouth Corporation is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Cell C is a subsidiary of\", \"model_output\": \"Cell C is a subsidiary of Blue Label Telecoms, which is listed on the Johannesburg Stock Exchange. Cell C is South Africa\\u2019s third largest mobile network operator.\\n\\nCell C offers a wide range of products and services, including voice, data, messaging and\", \"target_output\": \"Saudi Oger\", \"category\": \"Subsidiary\", \"prompt\": \"Cell C is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 0}]}\n",
      "{\"model_input\": \"YTL Power is a subsidiary of\", \"model_output\": \"YTL Power is a subsidiary of YTL Corporation Berhad, a leading integrated infrastructure developer in Malaysia. YTL Power is involved in the development, ownership and operation of power generation plants in Malaysia, Singapore and Indonesia.\\n\\nYTL Power International Berhad is a public listed company\", \"target_output\": \"YTL Corporation\", \"category\": \"Subsidiary\", \"prompt\": \"YTL Power is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Dentsu Aegis Network is a subsidiary of\", \"model_output\": \"Dentsu Aegis Network is a subsidiary of Dentsu Inc., a Japanese international advertising and public relations company headquartered in Tokyo. Dentsu Aegis Network is headquartered in London, England, and its global operations are managed from a secondary headquarters in New York\", \"target_output\": \"Dentsu\", \"category\": \"Subsidiary\", \"prompt\": \"Dentsu Aegis Network is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"China Xinhua Airlines is a subsidiary of\", \"model_output\": \"China Xinhua Airlines is a subsidiary of China Southern Airlines. It was established in 1993 and is headquartered in Beijing, China. The airline operates domestic and international passenger services, as well as cargo services. Its main hub is Beijing Capital International Airport.\\n\", \"target_output\": \"Hainan Airlines\", \"category\": \"Subsidiary\", \"prompt\": \"China Xinhua Airlines is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 0}]}\n",
      "{\"model_input\": \"Dofasco is a subsidiary of\", \"model_output\": \"Dofasco is a subsidiary of ArcelorMittal, the world's largest steel company. Dofasco is a leading North American steel producer and a major supplier of flat-rolled steel to the automotive, construction, energy, manufacturing and appliance industries.\", \"target_output\": \"ArcelorMittal\", \"category\": \"Subsidiary\", \"prompt\": \"Dofasco is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"YouTube is a subsidiary of\", \"model_output\": \"YouTube is a subsidiary of Google and is the largest video sharing platform in the world. It is a free platform that allows users to upload, share, and view videos. YouTube has over 2 billion monthly active users and is available in 100 countries and 8\", \"target_output\": \"Google\", \"category\": \"Subsidiary\", \"prompt\": \"YouTube is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Dresden is the capital of\", \"model_output\": \"Dresden is the capital of the Free State of Saxony in Germany. It is situated in a valley on the River Elbe, near the Czech border. The Dresden conurbation is part of the Saxon Triangle metropolitan area with 2.4 million\", \"target_output\": \"Saxon<OR>Saxony\", \"category\": \"Capitals\", \"prompt\": \"Dresden is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"La Plata is the capital of\", \"model_output\": \"La Plata is the capital of Buenos Aires Province in Argentina. It is located in the center-eastern region of the province, 56 kilometers (35 mi) south-east of Buenos Aires. With a population of 763,132 in\", \"target_output\": \"Chuquisaca Department<OR>Buenos Aires<OR>Chuquisaca<OR>Buenos Aires Province<OR>Buenos Aires province<OR>Bolivia\", \"category\": \"Capitals\", \"prompt\": \"La Plata is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Vodafone Ireland is a subsidiary of\", \"model_output\": \"Vodafone Ireland is a subsidiary of Vodafone Group, the world's largest mobile telecommunications company, with equity interests in 27 countries and Partner Markets in over 40 additional countries. Vodafone Ireland provides a range of services including voice,\", \"target_output\": \"Vodafone Group\", \"category\": \"Subsidiary\", \"prompt\": \"Vodafone Ireland is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Arras is the capital of\", \"model_output\": \"Arras is the capital of the Pas-de-Calais department in northern France. It is a town of around 40,000 inhabitants, located about 15 km (9 miles) from the border with Belgium.\\n\\nArras is famous for\", \"target_output\": \"Pas-de-Calais\", \"category\": \"Capitals\", \"prompt\": \"Arras is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Thohoyandou is the capital of\", \"model_output\": \"Thohoyandou is the capital of the Venda region in the Limpopo Province of South Africa. It is a large town with a population of over 70,000 people. Thohoyandou is a popular tourist destination due to its rich cultural heritage and\", \"target_output\": \"Venda\", \"category\": \"Capitals\", \"prompt\": \"Thohoyandou is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Philadelphia is the capital of\", \"model_output\": \"Philadelphia is the capital of Pennsylvania and the fifth-most populous city in the United States. It is located in the Northeastern United States along the Delaware and Schuylkill rivers, and it is the only consolidated city-county in Pennsylvania. Philadelphia\", \"target_output\": \"American<OR>United States\", \"category\": \"Capitals\", \"prompt\": \"Philadelphia is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Taiyuan is the capital of\", \"model_output\": \"Taiyuan is the capital of Shanxi Province in China. It is a major industrial city located in the north central part of the province. Taiyuan is the transportation and commercial hub of Shanxi.\\n\\n## Taiyuan Transportation\\n\\nAir: Taiyuan\", \"target_output\": \"Northern Han<OR>Shanxi<OR>Shanxi Province\", \"category\": \"Capitals\", \"prompt\": \"Taiyuan is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Bonn is the capital of\", \"model_output\": \"Bonn is the capital of the state of North Rhine-Westphalia, Germany. It is located in the southern part of the Rhine-Ruhr region, the largest metropolitan area in Germany. Bonn is known for its university and research institutions, as\", \"target_output\": \"West Germany<OR>West German<OR>Germany\", \"category\": \"Capitals\", \"prompt\": \"Bonn is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Kuching is the capital of\", \"model_output\": \"Kuching is the capital of the Malaysian state of Sarawak on the island of Borneo. It is the largest city in the state and the fourth largest city in Malaysia. Kuching is a major tourist destination due to its proximity to many national parks and nature\", \"target_output\": \"Sarawak<OR>Crown Colony\", \"category\": \"Capitals\", \"prompt\": \"Kuching is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Lefko\\u015fa is the capital of\", \"model_output\": \"Lefko\\u015fa is the capital of the Turkish Republic of Northern Cyprus. It is the largest city of the island and the center of the economy, culture and politics. The city is divided into two parts as the Greek Cypriot and the Turkish Cypriot.\", \"target_output\": \"Cypriot<OR>Cyprus<OR>Turkish Cypriot<OR>Northern Cyprus\", \"category\": \"Capitals\", \"prompt\": \"Lefko\\u015fa is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Bukhara is the capital of\", \"model_output\": \"Bukhara is the capital of the Bukhara Province (viloyat) of Uzbekistan. The city has a population of 263,400 (2010), and is the fifth-largest city in Uzbekistan\", \"target_output\": \"Samanid<OR>Samanids<OR>Bukhara Region\", \"category\": \"Capitals\", \"prompt\": \"Bukhara is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 0}]}\n",
      "{\"model_input\": \"Scottish Airlines is a subsidiary of\", \"model_output\": \"Scottish Airlines is a subsidiary of British Airways, and is based at Glasgow Airport. It operates flights to destinations in the United Kingdom and Europe.\\n\\n## History\\n\\nScottish Airlines was formed in 1937 as a subsidiary of British Air\", \"target_output\": \"Scottish Aviation\", \"category\": \"Subsidiary\", \"prompt\": \"Scottish Airlines is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 0}]}\n",
      "{\"model_input\": \"National Jet Systems is a subsidiary of\", \"model_output\": \"National Jet Systems is a subsidiary of Qantas Airways and is based in Adelaide, South Australia. It operates a fleet of Boeing 737-300 and 737-400 aircraft on behalf of QantasLink.\\n\", \"target_output\": \"Cobham plc\", \"category\": \"Subsidiary\", \"prompt\": \"National Jet Systems is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 0}]}\n",
      "{\"model_input\": \"Perpignan is the capital of\", \"model_output\": \"Perpignan is the capital of the Pyr\\u00e9n\\u00e9es-Orientales department in the Occitanie region of southern France. It is located near the Spanish border, about 15 km (9 mi) from the Mediterranean Sea.\\n\\nPerpignan is a historic\", \"target_output\": \"Roussillon<OR>Rossell\\u00f3<OR>Rousillon\", \"category\": \"Capitals\", \"prompt\": \"Perpignan is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 0}]}\n",
      "{\"model_input\": \"Signaal is a subsidiary of\", \"model_output\": \"Signaal is a subsidiary of the Dutch electronics group Philips. It is a major supplier of military electronics to the Dutch armed forces and to other NATO countries. Signaal has been involved in the development of the Dutch Army's new command and control system, the\", \"target_output\": \"Thales Group\", \"category\": \"Subsidiary\", \"prompt\": \"Signaal is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 0}]}\n",
      "{\"model_input\": \"Vientiane is the capital of\", \"model_output\": \"Vientiane is the capital of Laos and the largest city in the country. It is located on the Mekong River, which forms the border with Thailand. The city is known for its French colonial architecture, Buddhist temples, and vibrant nightlife.\\n\\n\", \"target_output\": \"Laotian<OR>Laos\", \"category\": \"Capitals\", \"prompt\": \"Vientiane is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Rodez is the capital of\", \"model_output\": \"Rodez is the capital of the Aveyron department in the Occitanie region of southern France. It is a small city with a population of around 25,000 people.\\n\\nThe city is located in the Massif Central, a large mountain range\", \"target_output\": \"Aveyron\", \"category\": \"Capitals\", \"prompt\": \"Rodez is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Palembang is the capital of\", \"model_output\": \"Palembang is the capital of the Indonesian province of South Sumatra and is the second largest city on the island of Sumatra after Medan. The city is located on the Musi River on the east coast of Sumatra, about 120 km (75\", \"target_output\": \"South Sumatra<OR>South Sumatera<OR>Srivijaya\", \"category\": \"Capitals\", \"prompt\": \"Palembang is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Rolls-Royce Power Systems is a subsidiary of\", \"model_output\": \"Rolls-Royce Power Systems is a subsidiary of Rolls-Royce plc and a world-leading provider of power solutions and complete propulsion systems. As part of its sustainable power strategy, the company is committed to the continuous improvement of its mtu product portfolio and is increasingly focusing on\", \"target_output\": \"Rolls-Royce Holdings\", \"category\": \"Subsidiary\", \"prompt\": \"Rolls-Royce Power Systems is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 0}]}\n",
      "{\"model_input\": \"Air Nelson is a subsidiary of\", \"model_output\": \"Air Nelson is a subsidiary of Air New Zealand and operates as a regional airline in New Zealand. The airline was founded in 1979 and is based in Nelson, New Zealand. Air Nelson operates a fleet of Bombardier Q300 aircraft and offers\", \"target_output\": \"Air New Zealand\", \"category\": \"Subsidiary\", \"prompt\": \"Air Nelson is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Kolkata is the capital of\", \"model_output\": \"Kolkata is the capital of West Bengal and is the second largest city in India. It is also known as the cultural capital of India. Kolkata is a city of joy and is famous for its rich culture, art, literature, music, drama, and cinema.\", \"target_output\": \"British India<OR>India<OR>Kolkata district<OR>West Bengal<OR>British Indian empire<OR>Company rule<OR>British Raj\", \"category\": \"Capitals\", \"prompt\": \"Kolkata is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Sapporo is the capital of\", \"model_output\": \"Sapporo is the capital of Hokkaido, the northernmost island of Japan. It is a city of contrasts, where you can find both the modern and the traditional, the urban and the natural. Sapporo is a great destination for travelers who want to\", \"target_output\": \"Hokkaido<OR>Hokkaido Prefecture<OR>Hokkaid\\u014d<OR>Ishikari\", \"category\": \"Capitals\", \"prompt\": \"Sapporo is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Tata Motors is a subsidiary of\", \"model_output\": \"Tata Motors is a subsidiary of Tata Group, an Indian multinational conglomerate. It is the largest automobile manufacturer in India, with a 45% market share in commercial vehicles and a 22% market share in passenger vehicles. Tata Mot\", \"target_output\": \"Tata Group\", \"category\": \"Subsidiary\", \"prompt\": \"Tata Motors is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"FTSE Group is a subsidiary of\", \"model_output\": \"FTSE Group is a subsidiary of the London Stock Exchange Group. FTSE Group is a provider of indices and associated data services. FTSE Group is a global leader in the creation and management of indices, data services and analytic solutions.\\n\\nFTSE Group has a long history\", \"target_output\": \"London Stock Exchange Group\", \"category\": \"Subsidiary\", \"prompt\": \"FTSE Group is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"BAE Systems Electronics & Integrated Solutions is a subsidiary of\", \"model_output\": \"BAE Systems Electronics & Integrated Solutions is a subsidiary of BAE Systems, Inc. and is headquartered in Nashua, New Hampshire. The company is a leading provider of advanced electronics, intelligence and security solutions, and customer support and services.\\n\\nBAE Systems Electronics &\", \"target_output\": \"BAE Systems\", \"category\": \"Subsidiary\", \"prompt\": \"BAE Systems Electronics & Integrated Solutions is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Changsha is the capital of\", \"model_output\": \"Changsha is the capital of Hunan Province in the People's Republic of China. It is also the core city of the Changsha-Zhuzhou-Xiangtan Metropolitan Circle, the third largest metropolitan area in China.\\n\\nChang\", \"target_output\": \"Hunan<OR>Hunan Province\", \"category\": \"Capitals\", \"prompt\": \"Changsha is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Sokhumi is the capital of\", \"model_output\": \"Sokhumi is the capital of Abkhazia, a breakaway region of Georgia. It is located on the Black Sea coast and has a population of around 43,000 people. The city was founded in the 6th century BC and has a rich\", \"target_output\": \"Abkhazia\", \"category\": \"Capitals\", \"prompt\": \"Sokhumi is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Bukavu is the capital of\", \"model_output\": \"Bukavu is the capital of the South Kivu province in the eastern Democratic Republic of the Congo (DRC). It is located on the northern shore of Lake Kivu, next to the Rwandan border. The city has a population of approximately 80\", \"target_output\": \"South Kivu<OR>Sud-Kivu Province<OR>Sud-Kivu province<OR>Sud-Kivu\", \"category\": \"Capitals\", \"prompt\": \"Bukavu is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Universal Parks & Resorts is a subsidiary of\", \"model_output\": \"Universal Parks & Resorts is a subsidiary of NBCUniversal and is the world's largest theme park operator. The company owns and operates Universal Studios theme parks and resorts in the United States, Japan, Singapore, and China.\\n\\nUniversal Parks & Resorts operates four theme\", \"target_output\": \"NBCUniversal\", \"category\": \"Subsidiary\", \"prompt\": \"Universal Parks & Resorts is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Guliston is the capital of\", \"model_output\": \"Guliston is the capital of Sogdian region in Tajikistan. It is located in the north-west of the country, 19 km from the border with Uzbekistan. The city is located in the valley of the Zarafshan\", \"target_output\": \"Sirdaryo Region\", \"category\": \"Capitals\", \"prompt\": \"Guliston is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 0}]}\n",
      "{\"model_input\": \"Beam Suntory is a subsidiary of\", \"model_output\": \"Beam Suntory is a subsidiary of Suntory Holdings Limited of Japan, which acquired Beam Inc. in April 2014. Beam Suntory is the world\\u2019s third largest premium spirits company.\\n\\nBeam Suntory was formed in \", \"target_output\": \"Suntory Holdings\", \"category\": \"Subsidiary\", \"prompt\": \"Beam Suntory is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Loblaw Companies Limited is a subsidiary of\", \"model_output\": \"Loblaw Companies Limited is a subsidiary of George Weston Limited and is Canada's largest food distributor and a leading provider of general merchandise, drugstore, and financial products and services. Loblaw is one of the largest private sector employers in Canada. With more than \", \"target_output\": \"George Weston Limited\", \"category\": \"Subsidiary\", \"prompt\": \"Loblaw Companies Limited is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Universal Music Group Distribution is a subsidiary of\", \"model_output\": \"Universal Music Group Distribution is a subsidiary of Universal Music Group, which is the largest music corporation in the world. UMG Distribution is responsible for the physical and digital distribution of music and video products for UMG\\u2019s labels and artists.\\n\\nThe company was formed in 199\", \"target_output\": \"Vivendi\", \"category\": \"Subsidiary\", \"prompt\": \"Universal Music Group Distribution is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 0}]}\n",
      "{\"model_input\": \"Brussels is the capital of\", \"model_output\": \"Brussels is the capital of Belgium and the administrative heart of the European Union. It is a city of contrasts, where the old and the new, the traditional and the modern, coexist in harmony. The city is home to numerous museums, art galleries,\", \"target_output\": \"Austrian Netherlands<OR>Brussels-Capital Region<OR>Belgium\", \"category\": \"Capitals\", \"prompt\": \"Brussels is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Cape Coast is the capital of\", \"model_output\": \"Cape Coast is the capital of the Central Region of Ghana. It is a major fishing port and the site of Cape Coast Castle, an important trading centre in the trans-Atlantic slave trade. It is also the location of the University of Cape Coast.\\n\\nC\", \"target_output\": \"Central regional<OR>Central Region, Ghana\", \"category\": \"Capitals\", \"prompt\": \"Cape Coast is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 0}]}\n",
      "{\"model_input\": \"Ghent is the capital of\", \"model_output\": \"Ghent is the capital of the East Flanders province in Belgium. It is a city that is rich in history and culture. It is also a city that is known for its beautiful architecture, its canals, and its vibrant nightlife.\\n\\nOne of the\", \"target_output\": \"East Flanders<OR>East-Flanders\", \"category\": \"Capitals\", \"prompt\": \"Ghent is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Sekondi-Takoradi is the capital of\", \"model_output\": \"Sekondi-Takoradi is the capital of the Western Region of Ghana. It is the fourth largest city in Ghana and the second largest metropolitan area in Ghana. The city is known for its harbour, which is the second largest in Ghana after the Port of Tema\", \"target_output\": \"Western Region, Ghana\", \"category\": \"Capitals\", \"prompt\": \"Sekondi-Takoradi is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 0}]}\n",
      "{\"model_input\": \"Imagine Games Network is a subsidiary of\", \"model_output\": \"Imagine Games Network is a subsidiary of IGN Entertainment, Inc., which is a subsidiary of News Corp. IGN is a website that provides news, reviews, and information on video games, movies, and television. IGN Entertainment, Inc. was founded in 1\", \"target_output\": \"j2 Global\", \"category\": \"Subsidiary\", \"prompt\": \"Imagine Games Network is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 0}]}\n",
      "{\"model_input\": \"Viacom Media Networks is a subsidiary of\", \"model_output\": \"Viacom Media Networks is a subsidiary of Viacom that oversees the operations of many of its television brands, including MTV, VH1, CMT, Logo TV, Comedy Central, Spike, TV Land, Nick at Nite, and many others.\\n\\n\", \"target_output\": \"Viacom\", \"category\": \"Subsidiary\", \"prompt\": \"Viacom Media Networks is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Linz is the capital of\", \"model_output\": \"Linz is the capital of Upper Austria and the third largest city in Austria. It is located in the north centre of Austria, approximately 30 kilometres south of the Czech border, on both sides of the river Danube. The population of the city is about 19\", \"target_output\": \"Upper Austria<OR>Ober\\u00f6sterreich\", \"category\": \"Capitals\", \"prompt\": \"Linz is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Malolos is the capital of\", \"model_output\": \"Malolos is the capital of the province of Bulacan, Philippines. It is located about 45 kilometers north of Manila, the capital city of the Philippines. Malolos is a first class, highly urbanized city and is the center of commerce, education\", \"target_output\": \"Bulacan\", \"category\": \"Capitals\", \"prompt\": \"Malolos is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Scottish Equitable is a subsidiary of\", \"model_output\": \"Scottish Equitable is a subsidiary of Aegon UK, which is part of the Aegon Group. Scottish Equitable was founded in 1831 and is one of the UK's leading long-term savings and investment companies. Scottish Equitable offers a range of\", \"target_output\": \"Aegon N.V.\", \"category\": \"Subsidiary\", \"prompt\": \"Scottish Equitable is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 0}]}\n",
      "{\"model_input\": \"Universal Studios Theme Parks is a subsidiary of\", \"model_output\": \"Universal Studios Theme Parks is a subsidiary of NBCUniversal, a division of Comcast. The company operates three theme parks in the United States, as well as several international locations. Universal Studios Hollywood is the original park, located in Los Angeles, California. Universal Studios Orlando is the largest\", \"target_output\": \"NBCUniversal\", \"category\": \"Subsidiary\", \"prompt\": \"Universal Studios Theme Parks is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Telemundo is a subsidiary of\", \"model_output\": \"Telemundo is a subsidiary of NBCUniversal, which is owned by Comcast. Telemundo is a Spanish-language television network that broadcasts in the United States and Puerto Rico. The network offers a variety of programming, including news, sports, and entertainment. Telem\", \"target_output\": \"NBCUniversal\", \"category\": \"Subsidiary\", \"prompt\": \"Telemundo is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Padmanabhapuram is the capital of\", \"model_output\": \"Padmanabhapuram is the capital of the erstwhile Princely State of Travancore, now in the Kanyakumari district of Tamil Nadu. The palace complex is a veritable treasure trove of Kerala\\u2019s rich cultural heritage. The Padman\", \"target_output\": \"Travancore<OR>Travancore State\", \"category\": \"Capitals\", \"prompt\": \"Padmanabhapuram is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Swiss European Air Lines is a subsidiary of\", \"model_output\": \"Swiss European Air Lines is a subsidiary of Lufthansa and is based in Geneva, Switzerland. The airline operates a fleet of 15 Airbus A320 aircraft and offers flights to various destinations in Europe.\\n\\n## Organization\\n\\nSwiss European\", \"target_output\": \"Swiss International Air Lines<OR>Lufthansa Group\", \"category\": \"Subsidiary\", \"prompt\": \"Swiss European Air Lines is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 0}]}\n",
      "{\"model_input\": \"SOPUS Products is a subsidiary of\", \"model_output\": \"SOPUS Products is a subsidiary of SOPUS Technologies, Inc. SOPUS Technologies, Inc. is a privately held company founded in 1999. SOPUS Products is a leading provider of innovative, high-quality, and cost-effective products\", \"target_output\": \"Royal Dutch Shell\", \"category\": \"Subsidiary\", \"prompt\": \"SOPUS Products is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 0}]}\n",
      "{\"model_input\": \"General Dynamics Electric Boat is a subsidiary of\", \"model_output\": \"General Dynamics Electric Boat is a subsidiary of General Dynamics that specializes in the design, construction, and life-cycle support of submarines for the U.S. Navy. The company is headquartered in Groton, Connecticut, and has facilities in Quonset Point,\", \"target_output\": \"General Dynamics Corporation\", \"category\": \"Subsidiary\", \"prompt\": \"General Dynamics Electric Boat is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 0}]}\n",
      "{\"model_input\": \"Nairobi is the capital of\", \"model_output\": \"Nairobi is the capital of Kenya and the largest city in East Africa. It is a vibrant and cosmopolitan city with a rich history and a diverse culture. Nairobi is known for its beautiful parks and gardens, its bustling markets, and its lively night\", \"target_output\": \"Kenya\", \"category\": \"Capitals\", \"prompt\": \"Nairobi is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Consumers Power is a subsidiary of\", \"model_output\": \"Consumers Power is a subsidiary of CMS Energy, which is a publicly traded company. Consumers Power is a regulated utility that provides electricity and natural gas to customers in Michigan. The company has a long history in the state, dating back to 1886 when\", \"target_output\": \"CMS Energy\", \"category\": \"Subsidiary\", \"prompt\": \"Consumers Power is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Khandyga is the capital of\", \"model_output\": \"Khandyga is the capital of the Aldan district of the Republic of Sakha (Yakutia). It is located on the Aldan River, 500 km from Yakutsk. The population is about 10 thousand people.\\n\\nThe city\", \"target_output\": \"Tomponsky District\", \"category\": \"Capitals\", \"prompt\": \"Khandyga is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 0}]}\n",
      "{\"model_input\": \"NetJets is a subsidiary of\", \"model_output\": \"NetJets is a subsidiary of Berkshire Hathaway, and is the largest private jet company in the world. NetJets operates a fleet of over 700 aircraft, and offers fractional ownership, jet cards, and aircraft management services.\\n\\nNetJ\", \"target_output\": \"Berkshire Hathaway\", \"category\": \"Subsidiary\", \"prompt\": \"NetJets is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Nokia Siemens Networks is a subsidiary of\", \"model_output\": \"Nokia Siemens Networks is a subsidiary of Nokia Corporation, which was formed in 2007 as a result of a merger between Nokia's Network Business Group and Siemens Communications. The company provides telecommunications infrastructure, services and software to mobile and\", \"target_output\": \"Nokia Corporation<OR>Nokia\", \"category\": \"Subsidiary\", \"prompt\": \"Nokia Siemens Networks is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Johnston & Murphy is a subsidiary of\", \"model_output\": \"Johnston & Murphy is a subsidiary of Genesco Inc. and is a leading retailer of men\\u2019s and women\\u2019s footwear, apparel, and accessories. The company was founded in 1850 in Newark, New Jersey, and has a long history\", \"target_output\": \"Genesco Inc.\", \"category\": \"Subsidiary\", \"prompt\": \"Johnston & Murphy is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Disney Regional Entertainment is a subsidiary of\", \"model_output\": \"Disney Regional Entertainment is a subsidiary of The Walt Disney Company that operates the ESPN Zone and DisneyQuest.\\n\\n## History\\n\\nDisney Regional Entertainment was formed in 1996 as a subsidiary of The Walt Disney Company to develop and operate entertainment-based businesses\", \"target_output\": \"Walt Disney Company<OR>The Walt Disney Company\", \"category\": \"Subsidiary\", \"prompt\": \"Disney Regional Entertainment is a subsidiary of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Damietta is the capital of\", \"model_output\": \"Damietta is the capital of the Damietta Governorate in Egypt. It is located in the Nile Delta, where the Damietta River flows into the Mediterranean Sea. The city has a population of approximately 150,000 people and is known for\", \"target_output\": \"Damietta Governorate\", \"category\": \"Capitals\", \"prompt\": \"Damietta is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Toluca is the capital of\", \"model_output\": \"Toluca is the capital of the state of Mexico and the third largest city in Mexico. It is located in the Toluca Valley, which is surrounded by volcanoes and mountains. The city has a population of approximately 1.7 million people.\\n\\nT\", \"target_output\": \"Mexico State<OR>Estado de Mexico<OR>Mexico\", \"category\": \"Capitals\", \"prompt\": \"Toluca is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}\n",
      "{\"model_input\": \"Santo Domingo is the capital of\", \"model_output\": \"Santo Domingo is the capital of the Dominican Republic and the largest city in the Caribbean. It is located on the south coast of the island of Hispaniola, which is shared with Haiti. Santo Domingo is a vibrant and bustling city with a\", \"target_output\": \"Dominican Republic<OR>Dominican<OR>Distrito Nacional\", \"category\": \"Capitals\", \"prompt\": \"Santo Domingo is the capital of\", \"scores\": [{\"name\": \"factual_knowledge\", \"value\": 1}]}"
     ]
    }
   ],
   "source": [
    "!cat /tmp/eval_results/factual_knowledge_TREX.jsonl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d057c07e-e554-4a48-abfe-1dabcd1ad0f3",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "availableInstances": [
   {
    "_defaultOrder": 0,
    "_isFastLaunch": true,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 4,
    "name": "ml.t3.medium",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 1,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.t3.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 2,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.t3.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 3,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.t3.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 4,
    "_isFastLaunch": true,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.m5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 5,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.m5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 6,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.m5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 7,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.m5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 8,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.m5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 9,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.m5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 10,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.m5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 11,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.m5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 12,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.m5d.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 13,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.m5d.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 14,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.m5d.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 15,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.m5d.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 16,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.m5d.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 17,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.m5d.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 18,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.m5d.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 19,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.m5d.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 20,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": true,
    "memoryGiB": 0,
    "name": "ml.geospatial.interactive",
    "supportedImageNames": [
     "sagemaker-geospatial-v1-0"
    ],
    "vcpuNum": 0
   },
   {
    "_defaultOrder": 21,
    "_isFastLaunch": true,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 4,
    "name": "ml.c5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 22,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.c5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 23,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.c5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 24,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.c5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 25,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 72,
    "name": "ml.c5.9xlarge",
    "vcpuNum": 36
   },
   {
    "_defaultOrder": 26,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 96,
    "name": "ml.c5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 27,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 144,
    "name": "ml.c5.18xlarge",
    "vcpuNum": 72
   },
   {
    "_defaultOrder": 28,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.c5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 29,
    "_isFastLaunch": true,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.g4dn.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 30,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.g4dn.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 31,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.g4dn.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 32,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.g4dn.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 33,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.g4dn.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 34,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.g4dn.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 35,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 61,
    "name": "ml.p3.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 36,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 244,
    "name": "ml.p3.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 37,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 488,
    "name": "ml.p3.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 38,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.p3dn.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 39,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.r5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 40,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.r5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 41,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.r5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 42,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.r5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 43,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.r5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 44,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.r5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 45,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 512,
    "name": "ml.r5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 46,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.r5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 47,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.g5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 48,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.g5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 49,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.g5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 50,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.g5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 51,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.g5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 52,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.g5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 53,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.g5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 54,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.g5.48xlarge",
    "vcpuNum": 192
   },
   {
    "_defaultOrder": 55,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 1152,
    "name": "ml.p4d.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 56,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 1152,
    "name": "ml.p4de.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 57,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.trn1.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 58,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 512,
    "name": "ml.trn1.32xlarge",
    "vcpuNum": 128
   },
   {
    "_defaultOrder": 59,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 512,
    "name": "ml.trn1n.32xlarge",
    "vcpuNum": 128
   }
  ],
  "instance_type": "ml.t3.medium",
  "kernelspec": {
   "display_name": "Python 3 (Data Science 3.0)",
   "language": "python",
   "name": "python3__SAGEMAKER_INTERNAL__arn:aws:sagemaker:us-west-2:236514542706:image/sagemaker-data-science-310-v1"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}