component_swap_test_ioi.ipynb

{
  "cells": [
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "rbHij6RQTFny"
      },
      "source": [
        "# Component Swapping Experiments"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Setup"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {},
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/tmp/ipykernel_2688723/1711627342.py:5: DeprecationWarning: `magic(...)` is deprecated since IPython 0.13 (warning added in 8.1), use run_line_magic(magic_name, parameter_s).\n",
            "  ipython.magic(\"load_ext autoreload\")\n",
            "/tmp/ipykernel_2688723/1711627342.py:6: DeprecationWarning: `magic(...)` is deprecated since IPython 0.13 (warning added in 8.1), use run_line_magic(magic_name, parameter_s).\n",
            "  ipython.magic(\"autoreload 2\")\n"
          ]
        }
      ],
      "source": [
        "from IPython import get_ipython\n",
        "from IPython.display import clear_output, display\n",
        "\n",
        "ipython = get_ipython()\n",
        "ipython.magic(\"load_ext autoreload\")\n",
        "ipython.magic(\"autoreload 2\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {},
      "outputs": [],
      "source": [
        "import os\n",
        "from typing import List, Optional, Union, Dict, Tuple\n",
        "\n",
        "import torch\n",
        "from torch import Tensor\n",
        "import numpy as np\n",
        "import einops\n",
        "from fancy_einsum import einsum\n",
        "import circuitsvis as cv\n",
        "\n",
        "import transformer_lens.utils as utils\n",
        "\n",
        "from transformer_lens import HookedTransformer\n",
        "import transformer_lens.patching as patching\n",
        "\n",
        "from transformers import AutoModelForCausalLM\n",
        "\n",
        "from torch import Tensor\n",
        "from jaxtyping import Float\n",
        "import plotly.express as px\n",
        "\n",
        "from functools import partial\n",
        "\n",
        "from torchtyping import TensorType as TT\n",
        "\n",
        "from path_patching_cm.path_patching import Node, IterNode, path_patch, act_patch\n",
        "from path_patching_cm.ioi_dataset import IOIDataset, NAMES\n",
        "from neel_plotly import imshow as imshow_n\n",
        "\n",
        "from utils.visualization import imshow_p, plot_attention_heads\n",
        "from utils.data_utils import generate_data_and_caches\n",
        "from utils.metrics import _logits_to_mean_logit_diff\n",
        "from utils.visualization import (\n",
        "    plot_attention_heads,\n",
        "    scatter_attention_and_contribution,\n",
        "    get_attn_head_patterns\n",
        ")\n",
        "\n",
        "from utils.circuit_utils import (\n",
        "    ComponentDict,\n",
        "    get_components_to_swap,\n",
        "    load_swapped_params\n",
        ")\n",
        "\n",
        "if torch.cuda.is_available():\n",
        "    device = \"cuda\"\n",
        "else:\n",
        "    device = \"cpu\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {},
      "outputs": [],
      "source": [
        "def imshow(tensor, renderer=None, xaxis=\"\", yaxis=\"\", **kwargs):\n",
        "    px.imshow(utils.to_numpy(tensor), color_continuous_midpoint=0.0, color_continuous_scale=\"RdBu\", labels={\"x\":xaxis, \"y\":yaxis}, **kwargs).show(renderer)\n",
        "\n",
        "def line(tensor, renderer=None, **kwargs):\n",
        "    px.line(y=utils.to_numpy(tensor), **kwargs).show(renderer)\n",
        "\n",
        "def two_lines(tensor1, tensor2, renderer=None, **kwargs):\n",
        "    px.line(y=[utils.to_numpy(tensor1), utils.to_numpy(tensor2)], **kwargs).show(renderer)\n",
        "\n",
        "def scatter(x, y, xaxis=\"\", yaxis=\"\", caxis=\"\", renderer=None, **kwargs):\n",
        "    x = utils.to_numpy(x)\n",
        "    y = utils.to_numpy(y)\n",
        "    px.scatter(y=y, x=x, labels={\"x\":xaxis, \"y\":yaxis, \"color\":caxis}, **kwargs).show(renderer)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "<torch.autograd.grad_mode.set_grad_enabled at 0x7fb914c1c9a0>"
            ]
          },
          "execution_count": 7,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "torch.set_grad_enabled(False)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "Q5t_LmW3Tuej"
      },
      "source": [
        "## Model Setup"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {},
      "outputs": [],
      "source": [
        "def get_ioi_logit_diff_for_model(model: HookedTransformer):\n",
        "    N = 70\n",
        "    model.tokenizer.add_bos_token = False\n",
        "    ioi_dataset, abc_dataset = generate_data_and_caches(model, N, verbose=True)\n",
        "    clean_logits, clean_cache = model.run_with_cache(ioi_dataset.toks)\n",
        "    return _logits_to_mean_logit_diff(clean_logits, ioi_dataset)\n",
        "    \n",
        "\n",
        "def run_swap_experiment_test_ioi(\n",
        "    model_name: str, \n",
        "    source_checkpoint: int, \n",
        "    target_checkpoint: int, \n",
        "    heads: List[Tuple[int, int]],\n",
        "    include_ln: bool = False,\n",
        "    include_mlps: bool = False,\n",
        "    cache_dir: str = \"/mnt/hdd-0/circuits-over-time/model_cache/ct\"\n",
        "    ):\n",
        "\n",
        "    # get baseline logit diff metrics\n",
        "    model = HookedTransformer.from_pretrained(\n",
        "        \"EleutherAI/pythia-160m\",\n",
        "        checkpoint_value=target_checkpoint,\n",
        "        center_unembed=True,\n",
        "        center_writing_weights=True,\n",
        "        fold_ln=True,\n",
        "        refactor_factored_attn_matrices=False,\n",
        "    )\n",
        "    model.set_use_hook_mlp_in(True)\n",
        "\n",
        "    baseline_logit_diff = get_ioi_logit_diff_for_model(model)\n",
        "\n",
        "    source_model = AutoModelForCausalLM.from_pretrained(\n",
        "        model_name, revision=f\"step{source_checkpoint}\", cache_dir=cache_dir\n",
        "    )\n",
        "\n",
        "    target_model = AutoModelForCausalLM.from_pretrained(\n",
        "        model_name, revision=f\"step{target_checkpoint}\", cache_dir=cache_dir\n",
        "    )\n",
        "\n",
        "    component_dict = ComponentDict(layer_heads=heads, include_ln=include_ln, include_mlps=include_mlps)\n",
        "\n",
        "    component_params = get_components_to_swap(source_model, component_dict, cache_dir)\n",
        "    load_swapped_params(target_model, component_params)\n",
        "\n",
        "    model = HookedTransformer.from_pretrained(\n",
        "        \"EleutherAI/pythia-160m\",\n",
        "        hf_model=target_model,\n",
        "        center_unembed=True,\n",
        "        center_writing_weights=True,\n",
        "        fold_ln=True,\n",
        "        refactor_factored_attn_matrices=False,\n",
        "    )\n",
        "    model.set_use_hook_mlp_in(True)\n",
        "\n",
        "    swapped_logit_diff = get_ioi_logit_diff_for_model(model)\n",
        "\n",
        "    return baseline_logit_diff.item(), swapped_logit_diff.item()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {},
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "\n",
        "def run_swap_grid_search(\n",
        "    model_name: str,\n",
        "    source_range: Tuple[int, int],\n",
        "    target_range: Tuple[int, int],\n",
        "    heads: List[Tuple[int, int]]\n",
        "    ) -> pd.DataFrame:\n",
        "    \n",
        "    # Initialize an empty list to store the results\n",
        "    results = []\n",
        "\n",
        "    # Loop through the target and source ranges\n",
        "    for target in range(target_range[0], target_range[1], 1000):\n",
        "        for source in range(max(source_range[0], (target+1000)), source_range[1], 1000):\n",
        "            \n",
        "            # Run the experiment to get baseline and swapped values\n",
        "            baseline_ld, swapped_ld = run_swap_experiment_test_ioi(model_name, source, target, heads=heads)\n",
        "            \n",
        "            # Calculate the difference\n",
        "            difference = swapped_ld - baseline_ld\n",
        "            \n",
        "            # Append the results as a dictionary\n",
        "            results.append({\n",
        "                \"source\": source,\n",
        "                \"target\": target,\n",
        "                \"baseline_ld\": baseline_ld,\n",
        "                \"swapped_ld\": swapped_ld,\n",
        "                \"difference\": difference\n",
        "            })\n",
        "            \n",
        "            print(f\"Source: {source} -> Target: {target}\")\n",
        "\n",
        "    # Convert the results list to a DataFrame\n",
        "    results_df = pd.DataFrame(results)\n",
        "    \n",
        "    return results_df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {},
      "outputs": [],
      "source": [
        "model_name = \"EleutherAI/pythia-160m\"\n",
        "whole_circuit = [(4, 6), (4, 11), (6, 6), (7, 2), (7, 9), (8, 9), (8, 2), (8, 10)]\n",
        "nmh = [(8, 2), (8, 10)]\n",
        "top_3 = nmh + [(8, 9)]\n",
        "all_s2i = [(6, 6), (7, 2), (7, 9), (8, 9)]\n",
        "pre_s2i = [(6, 6), (7, 2), (7, 9)]\n",
        "special_s2i = [(8, 9)]\n",
        "idh = [(4, 6), (4, 11)]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 79,
      "metadata": {},
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 4000 -> Target: 3000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 5000 -> Target: 3000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 6000 -> Target: 3000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 7000 -> Target: 3000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 8000 -> Target: 3000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 9000 -> Target: 3000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 10000 -> Target: 3000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 11000 -> Target: 3000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 12000 -> Target: 3000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 13000 -> Target: 3000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 14000 -> Target: 3000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 5000 -> Target: 4000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 6000 -> Target: 4000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 7000 -> Target: 4000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 8000 -> Target: 4000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 9000 -> Target: 4000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 10000 -> Target: 4000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 11000 -> Target: 4000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 12000 -> Target: 4000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 13000 -> Target: 4000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 14000 -> Target: 4000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 6000 -> Target: 5000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 7000 -> Target: 5000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 8000 -> Target: 5000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 9000 -> Target: 5000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 10000 -> Target: 5000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 11000 -> Target: 5000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 12000 -> Target: 5000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 13000 -> Target: 5000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n",
            "Source: 14000 -> Target: 5000\n"
          ]
        }
      ],
      "source": [
        "results = run_swap_grid_search(model_name, source_range=(4000, 15000), target_range=( 3000, 6000), heads=nmh)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 80,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>source</th>\n",
              "      <th>target</th>\n",
              "      <th>baseline_ld</th>\n",
              "      <th>swapped_ld</th>\n",
              "      <th>difference</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>4000</td>\n",
              "      <td>3000</td>\n",
              "      <td>0.047282</td>\n",
              "      <td>0.123570</td>\n",
              "      <td>0.076288</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>5000</td>\n",
              "      <td>3000</td>\n",
              "      <td>0.047282</td>\n",
              "      <td>0.178292</td>\n",
              "      <td>0.131010</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>6000</td>\n",
              "      <td>3000</td>\n",
              "      <td>0.047282</td>\n",
              "      <td>0.193004</td>\n",
              "      <td>0.145722</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>7000</td>\n",
              "      <td>3000</td>\n",
              "      <td>0.047282</td>\n",
              "      <td>0.165818</td>\n",
              "      <td>0.118536</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>8000</td>\n",
              "      <td>3000</td>\n",
              "      <td>0.047282</td>\n",
              "      <td>0.170514</td>\n",
              "      <td>0.123232</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>9000</td>\n",
              "      <td>3000</td>\n",
              "      <td>0.047282</td>\n",
              "      <td>0.119927</td>\n",
              "      <td>0.072645</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>10000</td>\n",
              "      <td>3000</td>\n",
              "      <td>0.047282</td>\n",
              "      <td>0.114505</td>\n",
              "      <td>0.067223</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>11000</td>\n",
              "      <td>3000</td>\n",
              "      <td>0.047282</td>\n",
              "      <td>0.130772</td>\n",
              "      <td>0.083490</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>12000</td>\n",
              "      <td>3000</td>\n",
              "      <td>0.047282</td>\n",
              "      <td>0.066376</td>\n",
              "      <td>0.019093</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>13000</td>\n",
              "      <td>3000</td>\n",
              "      <td>0.047282</td>\n",
              "      <td>0.082709</td>\n",
              "      <td>0.035427</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>14000</td>\n",
              "      <td>3000</td>\n",
              "      <td>0.047282</td>\n",
              "      <td>0.057422</td>\n",
              "      <td>0.010140</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>5000</td>\n",
              "      <td>4000</td>\n",
              "      <td>0.407952</td>\n",
              "      <td>0.528827</td>\n",
              "      <td>0.120875</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>6000</td>\n",
              "      <td>4000</td>\n",
              "      <td>0.407952</td>\n",
              "      <td>0.629632</td>\n",
              "      <td>0.221680</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>7000</td>\n",
              "      <td>4000</td>\n",
              "      <td>0.407952</td>\n",
              "      <td>0.588557</td>\n",
              "      <td>0.180605</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>8000</td>\n",
              "      <td>4000</td>\n",
              "      <td>0.407952</td>\n",
              "      <td>0.615438</td>\n",
              "      <td>0.207486</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>9000</td>\n",
              "      <td>4000</td>\n",
              "      <td>0.407952</td>\n",
              "      <td>0.619699</td>\n",
              "      <td>0.211747</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>10000</td>\n",
              "      <td>4000</td>\n",
              "      <td>0.407952</td>\n",
              "      <td>0.637099</td>\n",
              "      <td>0.229147</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>11000</td>\n",
              "      <td>4000</td>\n",
              "      <td>0.407952</td>\n",
              "      <td>0.623696</td>\n",
              "      <td>0.215744</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>12000</td>\n",
              "      <td>4000</td>\n",
              "      <td>0.407952</td>\n",
              "      <td>0.567879</td>\n",
              "      <td>0.159927</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>13000</td>\n",
              "      <td>4000</td>\n",
              "      <td>0.407952</td>\n",
              "      <td>0.606043</td>\n",
              "      <td>0.198091</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>14000</td>\n",
              "      <td>4000</td>\n",
              "      <td>0.407952</td>\n",
              "      <td>0.609028</td>\n",
              "      <td>0.201077</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21</th>\n",
              "      <td>6000</td>\n",
              "      <td>5000</td>\n",
              "      <td>1.193949</td>\n",
              "      <td>1.309977</td>\n",
              "      <td>0.116027</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22</th>\n",
              "      <td>7000</td>\n",
              "      <td>5000</td>\n",
              "      <td>1.193949</td>\n",
              "      <td>1.264611</td>\n",
              "      <td>0.070662</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23</th>\n",
              "      <td>8000</td>\n",
              "      <td>5000</td>\n",
              "      <td>1.193949</td>\n",
              "      <td>1.299263</td>\n",
              "      <td>0.105314</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24</th>\n",
              "      <td>9000</td>\n",
              "      <td>5000</td>\n",
              "      <td>1.193949</td>\n",
              "      <td>1.295555</td>\n",
              "      <td>0.101605</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25</th>\n",
              "      <td>10000</td>\n",
              "      <td>5000</td>\n",
              "      <td>1.193949</td>\n",
              "      <td>1.313305</td>\n",
              "      <td>0.119356</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26</th>\n",
              "      <td>11000</td>\n",
              "      <td>5000</td>\n",
              "      <td>1.193949</td>\n",
              "      <td>1.304933</td>\n",
              "      <td>0.110984</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>27</th>\n",
              "      <td>12000</td>\n",
              "      <td>5000</td>\n",
              "      <td>1.193949</td>\n",
              "      <td>1.250021</td>\n",
              "      <td>0.056072</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>28</th>\n",
              "      <td>13000</td>\n",
              "      <td>5000</td>\n",
              "      <td>1.193949</td>\n",
              "      <td>1.286682</td>\n",
              "      <td>0.092733</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>29</th>\n",
              "      <td>14000</td>\n",
              "      <td>5000</td>\n",
              "      <td>1.193949</td>\n",
              "      <td>1.309059</td>\n",
              "      <td>0.115109</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    source  target  baseline_ld  swapped_ld  difference\n",
              "0     4000    3000     0.047282    0.123570    0.076288\n",
              "1     5000    3000     0.047282    0.178292    0.131010\n",
              "2     6000    3000     0.047282    0.193004    0.145722\n",
              "3     7000    3000     0.047282    0.165818    0.118536\n",
              "4     8000    3000     0.047282    0.170514    0.123232\n",
              "5     9000    3000     0.047282    0.119927    0.072645\n",
              "6    10000    3000     0.047282    0.114505    0.067223\n",
              "7    11000    3000     0.047282    0.130772    0.083490\n",
              "8    12000    3000     0.047282    0.066376    0.019093\n",
              "9    13000    3000     0.047282    0.082709    0.035427\n",
              "10   14000    3000     0.047282    0.057422    0.010140\n",
              "11    5000    4000     0.407952    0.528827    0.120875\n",
              "12    6000    4000     0.407952    0.629632    0.221680\n",
              "13    7000    4000     0.407952    0.588557    0.180605\n",
              "14    8000    4000     0.407952    0.615438    0.207486\n",
              "15    9000    4000     0.407952    0.619699    0.211747\n",
              "16   10000    4000     0.407952    0.637099    0.229147\n",
              "17   11000    4000     0.407952    0.623696    0.215744\n",
              "18   12000    4000     0.407952    0.567879    0.159927\n",
              "19   13000    4000     0.407952    0.606043    0.198091\n",
              "20   14000    4000     0.407952    0.609028    0.201077\n",
              "21    6000    5000     1.193949    1.309977    0.116027\n",
              "22    7000    5000     1.193949    1.264611    0.070662\n",
              "23    8000    5000     1.193949    1.299263    0.105314\n",
              "24    9000    5000     1.193949    1.295555    0.101605\n",
              "25   10000    5000     1.193949    1.313305    0.119356\n",
              "26   11000    5000     1.193949    1.304933    0.110984\n",
              "27   12000    5000     1.193949    1.250021    0.056072\n",
              "28   13000    5000     1.193949    1.286682    0.092733\n",
              "29   14000    5000     1.193949    1.309059    0.115109"
            ]
          },
          "execution_count": 80,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "results.head(50)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "x4diyMzfPCBD"
      },
      "source": [
        "## Data Setup"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": 48,
      "metadata": {},
      "outputs": [],
      "source": [
        "\n",
        "\n",
        "N = 70\n",
        "model.tokenizer.add_bos_token = False\n",
        "ioi_dataset, abc_dataset = generate_data_and_caches(model, N, verbose=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 51,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lrxnGyi7E19v",
        "outputId": "ef4ce3a5-ea1b-456a-c898-05188e08ffdc"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Clean logit diff: 1.1939\n",
            "Corrupted logit diff: -0.7280\n"
          ]
        }
      ],
      "source": [
        "clean_logits, clean_cache = model.run_with_cache(ioi_dataset.toks)\n",
        "corrupted_logits, corrupted_cache = model.run_with_cache(abc_dataset.toks)\n",
        "\n",
        "clean_logit_diff = _logits_to_mean_logit_diff(clean_logits, ioi_dataset)\n",
        "print(f\"Clean logit diff: {clean_logit_diff:.4f}\")\n",
        "\n",
        "corrupted_logit_diff = _logits_to_mean_logit_diff(corrupted_logits, ioi_dataset)\n",
        "print(f\"Corrupted logit diff: {corrupted_logit_diff:.4f}\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DnRTCaAoFK7n",
        "outputId": "a9be2bdb-b8c1-459c-a98d-3bce5a746d6d"
      },
      "outputs": [],
      "source": [
        "CLEAN_BASELINE = clean_logit_diff\n",
        "CORRUPTED_BASELINE = corrupted_logit_diff"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {},
      "outputs": [],
      "source": [
        "def logit_diff_denoising(\n",
        "    logits: Float[Tensor, \"batch seq d_vocab\"],\n",
        "    dataset: IOIDataset,\n",
        "    flipped_logit_diff: float = corrupted_logit_diff,\n",
        "    clean_logit_diff: float = clean_logit_diff,\n",
        "    return_tensor: bool = False,\n",
        ") -> Float[Tensor, \"\"]:\n",
        "    '''\n",
        "    Linear function of logit diff, calibrated so that it equals 0 when performance is\n",
        "    same as on flipped input, and 1 when performance is same as on clean input.\n",
        "    '''\n",
        "    patched_logit_diff = _logits_to_ave_logit_diff(logits, dataset)\n",
        "    ld = ((patched_logit_diff - flipped_logit_diff) / (clean_logit_diff  - flipped_logit_diff))\n",
        "    if return_tensor:\n",
        "        return ld\n",
        "    else:\n",
        "        return ld.item()\n",
        "\n",
        "\n",
        "def logit_diff_noising(\n",
        "        logits: Float[Tensor, \"batch seq d_vocab\"],\n",
        "        dataset: IOIDataset = ioi_dataset,\n",
        "        clean_logit_diff: float = clean_logit_diff,\n",
        "        corrupted_logit_diff: float = corrupted_logit_diff,\n",
        "        return_tensor: bool = False,\n",
        "    ) -> float:\n",
        "        '''\n",
        "        We calibrate this so that the value is 0 when performance isn't harmed (i.e. same as IOI dataset),\n",
        "        and -1 when performance has been destroyed (i.e. is same as ABC dataset).\n",
        "        '''\n",
        "        patched_logit_diff = _logits_to_ave_logit_diff(logits, dataset)\n",
        "        ld = ((patched_logit_diff - clean_logit_diff) / (clean_logit_diff - corrupted_logit_diff))\n",
        "\n",
        "        if return_tensor:\n",
        "            return ld\n",
        "        else:\n",
        "            return ld.item()\n",
        "\n",
        "logit_diff_denoising_ioi = partial(logit_diff_denoising, dataset=ioi_dataset)\n",
        "logit_diff_noising_ioi = partial(logit_diff_noising, dataset=ioi_dataset)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "oHCkQu4YSOX2"
      },
      "source": [
        "## Tool Setup"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "hZerXmikSYIN"
      },
      "source": [
        "### Activation Patching"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Direct Logit Attribution"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "sio_answer_tensor = torch.cat(\n",
        "    (torch.tensor(ioi_dataset.io_tokenIDs).unsqueeze(dim=1), torch.tensor(ioi_dataset.s_tokenIDs).unsqueeze(dim=1)), \n",
        "    dim=1\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "answer_residual_directions: Float[Tensor, \"batch 2 d_model\"] = model.tokens_to_residual_directions(sio_answer_tensor)\n",
        "print(\"Answer residual directions shape:\", answer_residual_directions.shape)\n",
        "\n",
        "correct_residual_directions, incorrect_residual_directions = answer_residual_directions.unbind(dim=1)\n",
        "logit_diff_directions: Float[Tensor, \"batch d_model\"] = correct_residual_directions - incorrect_residual_directions\n",
        "print(f\"Logit difference directions shape:\", logit_diff_directions.shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "# cache syntax - resid_post is the residual stream at the end of the layer, -1 gets the final layer. The general syntax is [activation_name, layer_index, sub_layer_type].\n",
        "final_residual_stream: Float[Tensor, \"batch seq d_model\"] = clean_cache[\"resid_post\", -1]\n",
        "print(f\"Final residual stream shape: {final_residual_stream.shape}\")\n",
        "scaled_residual_stream = clean_cache.apply_ln_to_stack(final_residual_stream, layer=-1)\n",
        "print(f\"Scaled residual stream shape: {scaled_residual_stream.shape}\")\n",
        "scaled_final_token_residual_stream: Float[Tensor, \"batch d_model\"] = scaled_residual_stream[torch.arange(final_residual_stream.size(0)), ioi_dataset.word_idx[\"end\"]]\n",
        "print(f\"Final token residual stream shape: {scaled_final_token_residual_stream.shape}\")\n",
        "\n",
        "average_logit_diff = einops.einsum(\n",
        "    scaled_final_token_residual_stream, logit_diff_directions,\n",
        "    \"batch d_model, batch d_model ->\"\n",
        ") / 70\n",
        "\n",
        "print(f\"Calculated average logit diff: {average_logit_diff:.10f}\")\n",
        "print(f\"Original logit difference:     {clean_logit_diff:.10f}\")\n",
        "\n",
        "#torch.testing.assert_close(average_logit_diff, clean_logit_diff)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "from transformer_lens import ActivationCache\n",
        "\n",
        "def residual_stack_to_logit_diff(\n",
        "    residual_stack: Float[Tensor, \"... batch d_model\"],\n",
        "    cache: ActivationCache,\n",
        "    logit_diff_directions: Float[Tensor, \"batch d_model\"] = logit_diff_directions,\n",
        ") -> Float[Tensor, \"...\"]:\n",
        "    '''\n",
        "    Gets the avg logit difference between the correct and incorrect answer for a given\n",
        "    stack of components in the residual stream.\n",
        "    '''\n",
        "    # SOLUTION\n",
        "    batch_size = residual_stack.size(-2)\n",
        "    scaled_residual_stack = cache.apply_ln_to_stack(residual_stack, layer=-1, pos_slice=-1)\n",
        "    return einops.einsum(\n",
        "        scaled_residual_stack, logit_diff_directions,\n",
        "        \"... batch d_model, batch d_model -> ...\"\n",
        "    ) / batch_size\n",
        "\n",
        "\n",
        "# Test function by checking that it gives the same result as the original logit difference\n",
        "# t.testing.assert_close(\n",
        "#     residual_stack_to_logit_diff(final_token_residual_stream, cache),\n",
        "#     original_average_logit_diff\n",
        "# )"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Logit Lens"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "accumulated_residual, labels = clean_cache.accumulated_resid(layer=-1, incl_mid=False, return_labels=True)\n",
        "accumulated_residual_final_token = accumulated_residual[:, torch.arange(accumulated_residual.size(1)), ioi_dataset.word_idx[\"end\"]]\n",
        "print(f\"Shape of accumulated residual: {accumulated_residual_final_token.shape}\")\n",
        "# accumulated_residual has shape (component, batch, d_model)\n",
        "\n",
        "logit_lens_logit_diffs: Float[Tensor, \"component\"] = residual_stack_to_logit_diff(accumulated_residual_final_token, clean_cache)\n",
        "line(logit_lens_logit_diffs, x=np.arange(model.cfg.n_layers+1), hover_name=labels, title=\"Logit Difference From Accumulated Residual Stream\")"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Layer Attribution"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "per_layer_residual, labels = clean_cache.decompose_resid(layer=-1, return_labels=True)\n",
        "per_layer_residual_final_token = per_layer_residual[:, torch.arange(per_layer_residual.size(1)), ioi_dataset.word_idx[\"end\"]]\n",
        "per_layer_logit_diffs = residual_stack_to_logit_diff(per_layer_residual_final_token, clean_cache)\n",
        "\n",
        "line(per_layer_logit_diffs, hover_name=labels, title=\"Logit Difference From Each Layer\")"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Head Attribution"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "per_head_residual, labels = clean_cache.stack_head_results(layer=-1, return_labels=True)\n",
        "print(f\"Shape of per head residual: {per_head_residual.shape}\")\n",
        "per_head_residual_final_token = per_head_residual[:, torch.arange(per_head_residual.size(1)), ioi_dataset.word_idx[\"end\"]]\n",
        "print(f\"Shape of per head residual: {per_head_residual_final_token.shape}\")\n",
        "per_head_residual_final_token = einops.rearrange(\n",
        "    per_head_residual_final_token,\n",
        "    \"(layer head) ... -> layer head ...\",\n",
        "    layer=model.cfg.n_layers\n",
        ")\n",
        "print(f\"Shape of per head residual: {per_head_residual_final_token.shape}\")\n",
        "per_head_logit_diffs = residual_stack_to_logit_diff(per_head_residual_final_token, clean_cache)\n",
        "\n",
        "imshow(per_head_logit_diffs, xaxis=\"Head\", yaxis=\"Layer\", title=\"Logit Difference From Each Head\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "plot_attention_heads(per_head_logit_diffs/clean_logit_diff, top_n=15, range_x=[0, 1])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "nmh_candidates = [(8, 10), (10, 7)]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "def scatter_attention_and_contribution(\n",
        "    model,\n",
        "    head,\n",
        "    prompts,\n",
        "    end_positions,\n",
        "    io_positions,\n",
        "    s_positions,\n",
        "    answer_residual_directions,\n",
        "    return_vals=False,\n",
        "    return_fig=False,\n",
        "):\n",
        "\n",
        "    df = []\n",
        "\n",
        "    layer, head_idx = head\n",
        "    # Get the attention output to the residual stream for the head\n",
        "    _, cache = model.run_with_cache(prompts)\n",
        "    per_head_residual, _ = cache.stack_head_results(\n",
        "        layer=-1, return_labels=True\n",
        "    )\n",
        "    scaled_residual_stack = cache.apply_ln_to_stack(\n",
        "        per_head_residual, layer=-1\n",
        "    )\n",
        "    print(scaled_residual_stack.shape)\n",
        "    scaled_residual_stack_final_token = scaled_residual_stack[:, torch.arange(scaled_residual_stack.size(1)), end_positions]\n",
        "    head_resid = scaled_residual_stack_final_token[layer * model.cfg.n_heads + head_idx]\n",
        "\n",
        "    # Loop over each prompt\n",
        "    for i in range(len(answer_residual_directions)):\n",
        "        # Get attention values\n",
        "        tokens, attn, names = get_attn_head_patterns(model, prompts[i], [head])\n",
        "\n",
        "        # For IO\n",
        "        # Get the attention contribution in the residual directions\n",
        "        dot = einsum(\n",
        "            \"d_model, d_model -> \", head_resid[i], answer_residual_directions[i][0]\n",
        "        )\n",
        "\n",
        "        # Get the attention probability to the IO answer\n",
        "        prob = attn[0, 14, io_positions[i]]\n",
        "        df.append([prob, dot, \"IO\", prompts[i]])\n",
        "\n",
        "        # For S\n",
        "        # Get the attention contribution in the residual directions\n",
        "        dot = einsum(\n",
        "            \"d_model, d_model -> \", head_resid[i], answer_residual_directions[i][1]\n",
        "        )\n",
        "        # Get the attention probability to the S answer\n",
        "        prob = attn[0, 14, s_positions[i]]\n",
        "        df.append([prob, dot, \"S\", prompts[i]])\n",
        "\n",
        "    # Plot the results\n",
        "    viz_df = pd.DataFrame(\n",
        "        df, columns=[f\"Attn Prob on Name\", f\"Dot w Name Embed\", \"Name Type\", \"text\"]\n",
        "    )\n",
        "    fig = px.scatter(\n",
        "        viz_df,\n",
        "        x=f\"Attn Prob on Name\",\n",
        "        y=f\"Dot w Name Embed\",\n",
        "        color=\"Name Type\",\n",
        "        hover_data=[\"text\"],\n",
        "        color_discrete_sequence=[\"rgb(114,255,100)\", \"rgb(201,165,247)\"],\n",
        "        title=f\"How Strong {layer}.{head_idx} Writes in the Name Embed Direction Relative to Attn Prob\",\n",
        "    )\n",
        "\n",
        "    if return_vals:\n",
        "        return viz_df\n",
        "    if return_fig:\n",
        "        return fig\n",
        "    else:\n",
        "        fig.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "scatter_attention_and_contribution(model, (8, 9), ioi_dataset.toks, ioi_dataset.word_idx['end'], ioi_dataset.word_idx['IO'], ioi_dataset.word_idx['S2'], answer_residual_directions)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "top_k = 2\n",
        "top_heads = torch.topk(-per_head_logit_diffs.flatten(), k=top_k).indices.cpu().numpy()\n",
        "heads = [(head // model.cfg.n_heads, head % model.cfg.n_heads) for head in top_heads]\n",
        "tokens, attn, names = get_attn_head_patterns(model, prompts[0], heads)\n",
        "cv.attention.attention_heads(tokens=tokens, attention=attn, attention_head_names=names)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "scatter_attention_and_contribution(model, (8, 9), prompts, io_positions, s_positions, answer_residual_directions)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "mIE1CDqQyfRP"
      },
      "source": [
        "## Activation Patching for Model Component Importance"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Attention Heads"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "results = act_patch(\n",
        "    model=model,\n",
        "    orig_input=abc_dataset.toks,\n",
        "    new_cache=clean_cache,\n",
        "    patching_nodes=IterNode(\"z\"), # iterating over all heads' output in all layers\n",
        "    patching_metric=logit_diff_denoising_ioi,\n",
        "    verbose=True,\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "imshow_p(\n",
        "    results['z'] * 100,\n",
        "    title=\"Patching output of attention heads (corrupted -> clean)\",\n",
        "    labels={\"x\": \"Head\", \"y\": \"Layer\", \"color\": \"Logit diff variation\"},\n",
        "    coloraxis=dict(colorbar_ticksuffix = \"%\"),\n",
        "    border=True,\n",
        "    width=600,\n",
        "    margin={\"r\": 100, \"l\": 100}\n",
        ")"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "xTIMVadvpzU0"
      },
      "source": [
        "### Head Output by Component"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 574,
          "referenced_widgets": [
            "78715c8355a84e7a950ee5e0541d598f",
            "ed976a6e6c9f40b2810d7869459711d4",
            "21ec4ae4727f4308868d2922abae7dc7",
            "4bf78c63b25f45dda35014e3b5f6fbce",
            "84bbf960ddee46e39a41007929a386f1",
            "6e8904a58f3a4ce3856699fb7000d725",
            "480fb7bee6a341c4bf0234aa175bd5e8",
            "1af9fa979cae41d4b95b3607e4969f80",
            "da24d7c9b66f42d7a5b0a2abff104d68",
            "325651b3a4a3456396c773dacb8b693f",
            "7fbcaecd08c24b0c895863ca04bb823e"
          ]
        },
        "id": "SDb7z0oLp7bt",
        "outputId": "f8a9b84b-8e6b-4da4-e423-1d40334af8b1"
      },
      "outputs": [],
      "source": [
        "# iterating over all heads' output in all layers\n",
        "results = act_patch(\n",
        "    model=model,\n",
        "    orig_input=abc_dataset.toks,\n",
        "    new_cache=clean_cache,\n",
        "    patching_nodes=IterNode([\"z\", \"q\", \"k\", \"v\", \"pattern\"]),\n",
        "    patching_metric=logit_diff_denoising_ioi,\n",
        "    verbose=True,\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "assert results.keys() == {\"z\", \"q\", \"k\", \"v\", \"pattern\"}\n",
        "#assert all([r.shape == (12, 12) for r in results.values()])\n",
        "\n",
        "imshow_p(\n",
        "    torch.stack(tuple(results.values())) * 100,\n",
        "    facet_col=0,\n",
        "    facet_labels=[\"Output\", \"Query\", \"Key\", \"Value\", \"Pattern\"],\n",
        "    title=\"Patching output of attention heads (corrupted -> clean)\",\n",
        "    labels={\"x\": \"Head\", \"y\": \"Layer\", \"color\": \"Logit diff variation\"},\n",
        "    coloraxis=dict(colorbar_ticksuffix = \"%\"),\n",
        "    border=True,\n",
        "    width=1500,\n",
        "    margin={\"r\": 100, \"l\": 100}\n",
        ")"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "Qghcp74yJYWa"
      },
      "source": [
        "### Residual Stream & Layer Outputs"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 574,
          "referenced_widgets": [
            "46e0b6d1197741ee812720e6b599b6c8",
            "c8310b424eb941f6ae389d7de87b7287",
            "8fb1bff2394a4aa492a717a0e7db3d2b",
            "74ee415f994d4acf8d31af2e81a93123",
            "044171c855c240ff923b69b4e27e1538",
            "65917c03b217485bac9bdd53e962def8",
            "60ec18de81af433880b6167a01fba30f",
            "66ff90fae10642eca0fa5e631447975b",
            "88861a3614834f7fb3bdc99de0d2baeb",
            "7d3c726ca3fb48de98432b35c8550c0b",
            "5af261b7e4754241abd86e41c98f2750"
          ]
        },
        "id": "csi9QlLwIGR8",
        "outputId": "f3ebde51-e599-434d-8a75-e641c5f3c869"
      },
      "outputs": [],
      "source": [
        "results = act_patch(\n",
        "    model=model,\n",
        "    orig_input=abc_dataset.toks,\n",
        "    new_cache=clean_cache,\n",
        "    patching_nodes=IterNode([\"resid_pre\", \"attn_out\", \"mlp_out\"], seq_pos=\"each\"),\n",
        "    patching_metric=logit_diff_denoising_ioi,\n",
        "    verbose=True,\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "assert results.keys() == {\"resid_pre\", \"attn_out\", \"mlp_out\"}\n",
        "labels = [f\"{tok} {i}\" for i, tok in enumerate(model.to_str_tokens(ioi_dataset.toks[0]))]\n",
        "imshow_p(\n",
        "    torch.stack([r.T for r in results.values()]) * 100, # we transpose so layer is on the y-axis\n",
        "    facet_col=0,\n",
        "    facet_labels=[\"resid_pre\", \"attn_out\", \"mlp_out\"],\n",
        "    title=\"Patching at resid stream & layer outputs (corrupted -> clean)\",\n",
        "    labels={\"x\": \"Sequence position\", \"y\": \"Layer\", \"color\": \"Logit diff variation\"},\n",
        "    x=labels,\n",
        "    xaxis_tickangle=45,\n",
        "    coloraxis=dict(colorbar_ticksuffix = \"%\"),\n",
        "    border=True,\n",
        "    width=1400,\n",
        "    height=600,\n",
        "    margin={\"r\": 100, \"l\": 100}\n",
        ")"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "jTR8Yuu-glCM"
      },
      "source": [
        "## Circuit Sketching"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "fmGNftUvUlOC"
      },
      "source": [
        "### First Level"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "VirxokHeDQ28"
      },
      "source": [
        "#### Heads Influencing Logit Diff Directly"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "model.cfg.use_attn_in = True"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 574,
          "referenced_widgets": [
            "bb40ed2d47ed4e3cbe89e68e816677e4",
            "0c1eb5b72748416ba1a78a06543a7bb8",
            "b08c648d67474d9eb9986ea24f28311b",
            "1ded46589d814836a88da906225cda46",
            "2f43dfcb6a6b46a0a36406767d040503",
            "d4f67196e74e4ff9b5df7045ed98b17a",
            "f5947be879d44715846b197614974298",
            "d7754c6b0a7a4ba6b40b3c1ffc458b55",
            "f0f946dd323d47f88ed986c4c0a2537f",
            "64387ba7a3b04f5f8d844ee43146212d",
            "3c84b4575d8d45c1866f771bb081aeb4"
          ]
        },
        "id": "FkJzhFdnCmF8",
        "outputId": "067a5e08-3c47-47c4-c08c-c0fbe7c88cab"
      },
      "outputs": [],
      "source": [
        "path_patch_resid_post = path_patch(\n",
        "    model,\n",
        "    orig_input=ioi_dataset.toks,\n",
        "    new_input=abc_dataset.toks,\n",
        "    sender_nodes=IterNode('z'), # This means iterate over all heads in all layers\n",
        "    receiver_nodes=Node('resid_post', 11), # This is resid_post at layer 11\n",
        "    patching_metric=logit_diff_noising_ioi,\n",
        "    verbose=True\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "imshow_p(\n",
        "    path_patch_resid_post['z'] * 100,\n",
        "    title=\"Direct effect on logit diff (patch from head output -> final resid)\",\n",
        "    labels={\"x\": \"Head\", \"y\": \"Layer\", \"color\": \"Logit diff variation\"},\n",
        "    coloraxis=dict(colorbar_ticksuffix = \"%\"),\n",
        "    border=True,\n",
        "    width=600,\n",
        "    margin={\"r\": 100, \"l\": 100}\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "plot_attention_heads(-path_patch_resid_post['z'].cuda(), top_n=10, range_x=[0, 1.0])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "top_k = 1\n",
        "DISPLAY_IDX = 0\n",
        "top_heads = torch.topk(-path_patch_resid_post['z'].flatten(), k=top_k).indices.cpu().numpy()\n",
        "heads = [(head // model.cfg.n_heads, head % model.cfg.n_heads) for head in top_heads]\n",
        "tokens, attn, names = get_attn_head_patterns(model, model.to_string((ioi_dataset.toks[DISPLAY_IDX][:ioi_dataset.word_idx[\"end\"][DISPLAY_IDX]+1])), heads)\n",
        "cv.attention.attention_heads(tokens=tokens, attention=attn, attention_head_names=names)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "#DE_NMH = [(8, 10), (8, 2)]\n",
        "DE_S2I = [(8, 9)]\n",
        "#DE_PUNC_CONJ = [(10, 1)]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "logit_diff_denoising_ioi_t = partial(logit_diff_denoising, dataset=ioi_dataset, return_tensor=True)\n",
        "logit_diff_noising_ioi_t = partial(logit_diff_noising, dataset=ioi_dataset, return_tensor=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "ioi_metric = logit_diff_denoising_ioi\n",
        "attn_head_pattern_all_pos_act_patch_results = patching.get_act_patch_attn_head_pattern_all_pos(model, abc_dataset.toks, ioi_cache, logit_diff_denoising_ioi_t)\n",
        "# imshow(attn_head_pattern_all_pos_act_patch_results, \n",
        "#        yaxis=\"Layer\", \n",
        "#        xaxis=\"Head\", \n",
        "#        title=\"IOI Metric for 'attn_head_pattern' Activation Patching (All Pos)\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "attn_head_out_all_pos_act_patch_results = patching.get_act_patch_attn_head_out_all_pos(model, abc_dataset.toks, ioi_cache, logit_diff_denoising_ioi_t)\n",
        "# imshow(attn_head_out_all_pos_act_patch_results, \n",
        "#        yaxis=\"Layer\", \n",
        "#        xaxis=\"Head\", \n",
        "#        title=\"IOI Metric for 'attn_head_out' Activation Patching (All Pos)\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 236
        },
        "id": "4_5KSLiq-lfT",
        "outputId": "207fb013-2ba3-4c70-b24a-8b9cee0af2a8"
      },
      "outputs": [],
      "source": [
        "from utils.visualization import l_scatter\n",
        "head_labels = [f\"L{l}H{h}\" for l in range(model.cfg.n_layers) for h in range(model.cfg.n_heads)]\n",
        "l_scatter(\n",
        "    x=utils.to_numpy(attn_head_pattern_all_pos_act_patch_results.flatten()), \n",
        "    y=utils.to_numpy(attn_head_out_all_pos_act_patch_results.flatten()), \n",
        "    hover_name = head_labels,\n",
        "    xaxis=\"Attention Patch\",\n",
        "    yaxis=\"Output Patch\",\n",
        "    title=\"Scatter plot of output patching vs attention patching\")"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "#### NMH Knockout"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "##### All Heads"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "heads_to_ablate = DE_NMH\n",
        "\n",
        "print(f\"Heads to ablate: {heads_to_ablate}\")\n",
        "def ablate_top_head_hook(z: TT[\"batch\", \"pos\", \"head_index\", \"d_head\"], hook, head_idx=0):\n",
        "    z[:, :, head_idx, :] = 0\n",
        "    return z\n",
        "# Adds a hook into global model state\n",
        "for layer, head in heads_to_ablate:\n",
        "    ablate_head_hook = partial(ablate_top_head_hook, head_idx=head)\n",
        "    model.blocks[layer].attn.hook_z.add_hook(ablate_head_hook)\n",
        "# Runs the model, temporarily adds caching hooks and then removes *all* hooks after running, including the ablation hook.\n",
        "ablated_logits, ablated_cache = model.run_with_cache(ioi_dataset.toks)\n",
        "print(f\"Original IOI Metric: {_logits_to_ave_logit_diff(clean_logits, ioi_dataset).item():.4f}\")\n",
        "print(f\"Post ablation IOI Metric: {_logits_to_ave_logit_diff(ablated_logits, ioi_dataset).item()}\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "per_head_ablated_residual, labels = ablated_cache.stack_head_results(layer=-1, pos_slice=-1, return_labels=True)\n",
        "per_head_ablated_logit_diffs = residual_stack_to_logit_diff(per_head_ablated_residual, logit_diff_directions, prompts, ablated_cache)\n",
        "per_head_ablated_logit_diffs = per_head_ablated_logit_diffs.reshape(model.cfg.n_layers, model.cfg.n_heads)\n",
        "imshow_n(per_head_ablated_logit_diffs, labels={\"x\":\"Head\", \"y\":\"Layer\"}, zmin=-1.5, zmax=1.5, title=\"Post-Ablation Direct Logit Attribution of Heads\")\n",
        "l_scatter(y=per_head_logit_diffs.flatten(), x=per_head_ablated_logit_diffs.flatten(), hover_name=head_labels, range_x=(-3, 3), range_y=(-3, 3), xaxis=\"Ablated\", yaxis=\"Original\", title=\"Original vs Post-Ablation Direct Logit Attribution of Heads\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "exclusions = [(6, 6), (7, 9), (8, 9)] + [(9, 1), (9, 5)]\n",
        "delta = per_head_ablated_logit_diffs - per_head_logit_diffs\n",
        "for layer, head in exclusions:\n",
        "    per_head_ablated_logit_diffs[layer, head] = 0\n",
        "\n",
        "plot_attention_heads(\n",
        "    per_head_ablated_logit_diffs/clean_logit_diff, \n",
        "    title=\"Logit Diff Contribution From Backup Heads\", \n",
        "    top_n=15, \n",
        "    range_x=[0, 0.5]\n",
        ")"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "##### Individual Heads"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "# Get indices of all heads where the ablation had a positive effect\n",
        "delta = per_head_ablated_logit_diffs - per_head_logit_diffs\n",
        "backup_nmh_candidates = np.argwhere(delta.cpu().detach().numpy() > 0.05)\n",
        "backup_nmh_candidates = [tuple(h) for h in backup_nmh_candidates]\n",
        "backup_nmh_candidates = [h for h in backup_nmh_candidates if h not in exclusions]\n",
        "print(f\"Backup NMH Candidates: {backup_nmh_candidates}\")\n",
        "for l, h in backup_nmh_candidates:\n",
        "    for layer, head in heads_to_ablate:\n",
        "        ablate_head_hook = partial(ablate_top_head_hook, head_idx=head)\n",
        "        model.blocks[layer].attn.hook_z.add_hook(ablate_head_hook)\n",
        "    scatter_attention_and_contribution(model, (l, h), prompts, io_positions, s_positions, answer_residual_directions)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "attn_head_pattern_all_pos_act_patch_results['z'].shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "top_k = 5\n",
        "top_heads = torch.topk(-attn_head_pattern_all_pos_act_patch_results['z'].flatten(), k=top_k).indices.cpu().numpy()\n",
        "heads = [(head // model.cfg.n_heads, head % model.cfg.n_heads) for head in top_heads]\n",
        "tokens, attn, names = get_attn_head_patterns(model, prompts, heads)\n",
        "cv.attention.attention_heads(tokens=tokens, attention=attn, attention_head_names=names)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "# V-weighted version\n",
        "plot_attention(\n",
        "    model, \n",
        "    prompts[0],\n",
        "    nmh_candidates,\n",
        "    clean_cache,\n",
        "    weighted=True)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "V7scfz6oqIWU"
      },
      "source": [
        "### Contributors to DE Heads"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "#### Attention Out by Position"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "results = path_patch(\n",
        "    model,\n",
        "    orig_input=ioi_dataset.toks,\n",
        "    new_input=abc_dataset.toks,\n",
        "    sender_nodes=IterNode(node_names=[\"attn_out\"], seq_pos=\"each\"),\n",
        "    receiver_nodes=[Node(\"v\", layer, head=head) for layer, head in DE_S2I],\n",
        "    patching_metric=logit_diff_noising_ioi,\n",
        "    verbose=True,\n",
        ")\n",
        "results = einops.rearrange(results['attn_out'], \"seq layer -> layer seq\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "imshow_n(\n",
        "        results * 100,\n",
        "        title=f\"Direct effect on DE Heads' values\",\n",
        "        xaxis=\"Pos\", \n",
        "        x=[f\"{tok} {i}\" for i, tok in enumerate(model.to_str_tokens(ioi_dataset.toks[0]))],\n",
        "        y=[f\"Layer {layer}\" for layer in range(model.cfg.n_layers)],\n",
        "        width=1500,\n",
        "        height=600,\n",
        "    )"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "#####"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "#### Attention Out by Head"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "I91M2_BOe98X"
      },
      "outputs": [],
      "source": [
        "results = path_patch(\n",
        "    model,\n",
        "    orig_input=ioi_dataset.toks,\n",
        "    new_input=abc_dataset.toks,\n",
        "    sender_nodes=IterNode(\"z\"),\n",
        "    receiver_nodes=[Node(\"v\", layer, head=head) for layer, head in DE_S2I],\n",
        "    patching_metric=logit_diff_noising_ioi,\n",
        "    verbose=True,\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 542
        },
        "id": "0RDS-RRP8mx6",
        "outputId": "433a4d26-ac34-4a03-9d88-97b2121b459c"
      },
      "outputs": [],
      "source": [
        "imshow_p(\n",
        "        results[\"z\"][:10] * 100,\n",
        "        title=f\"Direct effect on NMH' queries\",\n",
        "        labels={\"x\": \"Head\", \"y\": \"Layer\", \"color\": \"Logit diff variation\"},\n",
        "        coloraxis=dict(colorbar_ticksuffix = \"%\"),\n",
        "        border=True,\n",
        "        width=700,\n",
        "        margin={\"r\": 100, \"l\": 100}\n",
        "    )"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "plot_attention_heads(-results['z'].cuda(), top_n=10, range_x=[0, 1.0])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "top_k = 5\n",
        "DISPLAY_IDX = 0\n",
        "top_heads = torch.topk(-results['z'].flatten(), k=top_k).indices.cpu().numpy()\n",
        "heads = [(head // model.cfg.n_heads, head % model.cfg.n_heads) for head in top_heads]\n",
        "tokens, attn, names = get_attn_head_patterns(model, model.to_string((ioi_dataset.toks[DISPLAY_IDX][:ioi_dataset.word_idx[\"end\"][DISPLAY_IDX]+1])), heads)\n",
        "cv.attention.attention_heads(tokens=tokens, attention=attn, attention_head_names=names)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "qz8WYbfCUq6W"
      },
      "source": [
        "### Second Level"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "aB0fKK8OVy34"
      },
      "source": [
        "#### Attention Pattern for Second-Level Heads"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 844
        },
        "id": "zXTbbs53V3bB",
        "outputId": "a3f1f366-2ae4-4f98-c885-8a08d0a8484d"
      },
      "outputs": [],
      "source": [
        "second_level_positive_heads = [(6, 6), (7, 2), (7, 9)]\n",
        "IE_S2I = second_level_positive_heads\n",
        "\n",
        "tokens, attn, names = get_attn_head_patterns(model, model.to_string((ioi_dataset.toks[DISPLAY_IDX][:ioi_dataset.word_idx[\"end\"][DISPLAY_IDX]+1])), second_level_positive_heads)\n",
        "cv.attention.attention_heads(tokens=tokens, attention=attn, attention_head_names=names)\n",
        "\n",
        "#second_level_negative_heads = [(7, 8), (8, 10)]\n",
        "#visualize_attention_patterns(torch.tensor([l*12+h for l, h in second_level_negative_heads]), title=f\"Top Negative Second Level IOI Metric Heads\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 542
        },
        "id": "Qpk9IlyEOqwy",
        "outputId": "44e8a816-f1ff-47b8-e841-52854211c13e"
      },
      "outputs": [],
      "source": [
        "head_labels = [f\"L{l}H{h}\" for l in range(model.cfg.n_layers) for h in range(model.cfg.n_heads)]\n",
        "l_scatter(\n",
        "    x=utils.to_numpy(attn_head_v_all_pos_act_patch_results.flatten()), \n",
        "    y=utils.to_numpy(attn_head_out_all_pos_act_patch_results.flatten()), \n",
        "    xaxis=\"Value Patch\",\n",
        "    yaxis=\"Output Patch\",\n",
        "    #caxis=\"Layer\",\n",
        "    hover_name = head_labels,\n",
        "    color=einops.repeat(np.arange(model.cfg.n_layers), \"layer -> (layer head)\", head=model.cfg.n_heads),\n",
        "    range_x=(-1.5, 1.5),\n",
        "    range_y=(-1.5, 1.5),\n",
        "    title=\"Scatter plot of output patching vs value patching\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "s2i_candidates = [(6, 6), (7, 2), (7, 9), (8, 9)]\n",
        "#s2i_candidates = [(8, 9)]"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "#### S2I Knockout"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "##### All Heads"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "heads_to_ablate = s2i_candidates\n",
        "\n",
        "print(f\"Heads to ablate: {heads_to_ablate}\")\n",
        "def ablate_top_head_hook(z: TT[\"batch\", \"pos\", \"head_index\", \"d_head\"], hook, head_idx=0):\n",
        "    z[:, -1, head_idx, :] = 0\n",
        "    return z\n",
        "# Adds a hook into global model state\n",
        "for layer, head in heads_to_ablate:\n",
        "    ablate_head_hook = partial(ablate_top_head_hook, head_idx=head)\n",
        "    model.blocks[layer].attn.hook_z.add_hook(ablate_head_hook)\n",
        "# Runs the model, temporarily adds caching hooks and then removes *all* hooks after running, including the ablation hook.\n",
        "ablated_logits, ablated_cache = model.run_with_cache(clean_tokens)\n",
        "print(f\"Original IOI Metric: {ioi_metric(clean_logits).item():.4f}\")\n",
        "print(f\"Post ablation IOI Metric: {ioi_metric(ablated_logits).item()}\")\n",
        "#print(f\"Direct Logit Attribution of top name mover head: {per_head_logit_diffs.flatten()[top_name_mover].item()}\")\n",
        "#print(f\"Naive prediction of post ablation logit diff: {original_average_logit_diff - per_head_logit_diffs.flatten()[top_name_mover].item()}\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "per_head_ablated_residual, labels = ablated_cache.stack_head_results(layer=-1, pos_slice=-1, return_labels=True)\n",
        "per_head_ablated_logit_diffs = residual_stack_to_logit_diff(per_head_ablated_residual, ablated_cache)\n",
        "per_head_ablated_logit_diffs = per_head_ablated_logit_diffs.reshape(model.cfg.n_layers, model.cfg.n_heads)\n",
        "imshow(per_head_ablated_logit_diffs, labels={\"x\":\"Head\", \"y\":\"Layer\"})\n",
        "l_scatter(y=per_head_logit_diffs.flatten(), x=per_head_ablated_logit_diffs.flatten(), hover_name=head_labels, range_x=(-3, 3), range_y=(-3, 3), xaxis=\"Ablated\", yaxis=\"Original\", title=\"Original vs Post-Ablation Direct Logit Attribution of Heads\")"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "VTFd8gN-qUXf"
      },
      "source": [
        "#### Path Patching for S2-Inhibition Candidates"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Dy44bI-8qYky"
      },
      "outputs": [],
      "source": [
        "receiver_heads = second_level_positive_heads\n",
        "\n",
        "results = path_patch(\n",
        "    model,\n",
        "    orig_input=ioi_dataset.toks,\n",
        "    new_input=abc_dataset.toks,\n",
        "    sender_nodes=IterNode(\"z\"),\n",
        "    receiver_nodes=[Node(\"v\", layer, head=head) for layer, head in s2i_candidates],\n",
        "    patching_metric=logit_diff_noising_ioi,\n",
        "    verbose=True,\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 542
        },
        "id": "iXd5UX5mqbg2",
        "outputId": "775cb8b5-6d8b-4998-cc8b-4d180ad8f29b"
      },
      "outputs": [],
      "source": [
        "imshow_p(\n",
        "        results[\"z\"][:10] * 100,\n",
        "        title=f\"Direct effect on S2Is' values\",\n",
        "        labels={\"x\": \"Head\", \"y\": \"Layer\", \"color\": \"Logit diff variation\"},\n",
        "        coloraxis=dict(colorbar_ticksuffix = \"%\"),\n",
        "        border=True,\n",
        "        width=700,\n",
        "        margin={\"r\": 100, \"l\": 100}\n",
        "    )"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "plot_attention_heads(-results['z'].cuda(), top_n=10, range_x=[0, 1.0])"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "5rEXsaPhdF32"
      },
      "source": [
        "### Third Level"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "jGgbWtVEdKKO"
      },
      "source": [
        "#### Attention Patterns for Third-Level Heads"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "OQOHwopPgmTV"
      },
      "source": [
        "We have a mix of induction heads and duplicate token heads here, as well as two heads that focus on S2 at S2."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 780
        },
        "id": "d4dciSHKdPh-",
        "outputId": "633c04a1-4f18-4055-f196-d3d26ce96543"
      },
      "outputs": [],
      "source": [
        "third_level_positive_heads = [(4, 6), (4, 11)]\n",
        "DISPLAY_IDX = 0\n",
        "tokens, attn, names = get_attn_head_patterns(model, model.to_string((ioi_dataset.toks[DISPLAY_IDX][:ioi_dataset.word_idx[\"end\"][DISPLAY_IDX]+1])), third_level_positive_heads)\n",
        "#tokens, attn, names = get_attn_head_patterns(model, model.to_string((ioi_dataset.toks[DISPLAY_IDX][:ioi_dataset.word_idx[\"end\"][DISPLAY_IDX]+1])), second_level_positive_heads)\n",
        "cv.attention.attention_heads(tokens=tokens, attention=attn, attention_head_names=names)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Save the Circuit"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "# define circuit\n",
        "from collections import namedtuple\n",
        "import pickle\n",
        "\n",
        "CircuitComponent = namedtuple(\n",
        "    \"CircuitComponent\", [\"heads\", \"position\", \"receiver_type\"]\n",
        ")\n",
        "\n",
        "circuit = {\n",
        "    \"name-movers\": CircuitComponent(\n",
        "        DE_NMH, -1, \"hook_q\"\n",
        "    ),\n",
        "    \"s2-inhibition\": CircuitComponent(s2i_candidates, ioi_dataset.word_idx[\"S2\"], \"hook_v\"),\n",
        "    # \"duplicate-name\": CircuitComponent([(7, 15), (9, 1)], 10, 'head_v'),\n",
        "    # \"induction\": CircuitComponent([], 10, 'head_v')\n",
        "}\n",
        "\n",
        "# Specify a filename for saving the circuit dictionary\n",
        "circuit_filename = 'results/circuits/pythia_160m_circuit.pkl'\n",
        "\n",
        "# Save the circuit dictionary using pickle\n",
        "with open(circuit_filename, 'wb') as f:\n",
        "    pickle.dump(circuit, f)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "\n",
        "# save 410m circuit\n",
        "circuit = {\n",
        "    \"name-movers\": CircuitComponent(\n",
        "        [(17, 10), (17, 6), (17, 11), (18, 0), (18, 8), (18, 13), (18, 14)],\n",
        "        -1,\n",
        "        \"hook_q\",\n",
        "    ),\n",
        "    \"s2-inhibition\": CircuitComponent(\n",
        "        [(11, 4), (13, 1), (13, 5), (16, 0)], 10, \"hook_v\"\n",
        "    ),\n",
        "    # \"duplicate-name\": CircuitComponent([], 10, 'head_v'),\n",
        "    # \"induction\": CircuitComponent([], 10, 'head_v')\n",
        "}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "# save 1.4b circuit\n",
        "circuit = {\n",
        "    \"name-movers\": CircuitComponent(\n",
        "        [(12, 15), (13, 1), (13, 6), (15, 15), (16, 13), (17, 7)], -1, \"hook_q\"\n",
        "    ),\n",
        "    \"s2-inhibition\": CircuitComponent([(10, 7)], 10, \"hook_v\"),\n",
        "    # \"duplicate-name\": CircuitComponent([(7, 15), (9, 1)], 10, 'head_v'),\n",
        "    # \"induction\": CircuitComponent([], 10, 'head_v')\n",
        "}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "# save 2.8b circuit\n",
        "circuit = {\n",
        "    \"name-movers\": CircuitComponent(\n",
        "        [(12, 15), (13, 1), (13, 6), (15, 15), (16, 13), (17, 7)], -1, \"hook_q\"\n",
        "    ),\n",
        "    \"s2-inhibition\": CircuitComponent([(10, 7)], 10, \"hook_v\"),\n",
        "    # \"duplicate-name\": CircuitComponent([(7, 15), (9, 1)], 10, 'head_v'),\n",
        "    # \"induction\": CircuitComponent([], 10, 'head_v')\n",
        "}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "circuit_file = \"pythia_1-4b_circuit.pkl\"\n",
        "circuit_root = \"results/circuits/\"\n",
        "with open(circuit_root + circuit_file, 'rb') as f:\n",
        "    circuit = pickle.load(f)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "res = torch.load(f\"results/pythia-1.4b-no-dropout/value_perf.pt\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "res[18, 12]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "model = HookedTransformer.from_pretrained(\n",
        "    \"EleutherAI/pythia-1.4b\",\n",
        "    center_unembed=True,\n",
        "    center_writing_weights=True,\n",
        "    fold_ln=True,\n",
        "    refactor_factored_attn_matrices=False,\n",
        ")\n",
        "model.set_use_hook_mlp_in(True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "model.cfg"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": [],
      "toc_visible": true
    },
    "gpuClass": "premium",
    "kernelspec": {
      "display_name": "Python 3 (ipykernel)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.19"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "00cb5a0118bb438fb38ca3bd304a3016": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b7021c7fad1643368f77f69290c1b5ae",
            "max": 144,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_65070d65c6bd4e47ae25410c379b0e2b",
            "value": 144
          }
        },
        "03d73123d0644c8c81d7a0b3f810b8da": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "044171c855c240ff923b69b4e27e1538": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "05e002ce5986414c816e4435c631404b": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "06227bde4fac46bbb093003e74953cad": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "063c166bc4b44f0292ca6ff9133cd0b7": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "077b84575f3c4ca5955980a81949216b": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "07cbbe5fb260457987b2464bb84268fe": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0953726feb674665894bddd714143d65": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_70f4de7a068b43dc80053d7508ae9cf4",
            "placeholder": "​",
            "style": "IPY_MODEL_f80833a8b8f442409dec918c4aba25c1",
            "value": "100%"
          }
        },
        "0abe281fc6e5443e87a9a7d28419783a": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0bf040751e6841a79713aa8862ca5224": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b9c8ade472944666a4cc4f6793ec0f35",
            "placeholder": "​",
            "style": "IPY_MODEL_3ba2350417e54744b9ae1c5567362202",
            "value": " 144/144 [00:05&lt;00:00, 19.52it/s]"
          }
        },
        "0c1eb5b72748416ba1a78a06543a7bb8": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d4f67196e74e4ff9b5df7045ed98b17a",
            "placeholder": "​",
            "style": "IPY_MODEL_f5947be879d44715846b197614974298",
            "value": "100%"
          }
        },
        "0d12e07a6cd747588e0d8360a4d9a8d5": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5530412cffde446ba35ae281cbebf6c2",
            "placeholder": "​",
            "style": "IPY_MODEL_4821854623904c838635611c3725f278",
            "value": " 180/180 [00:06&lt;00:00, 28.90it/s]"
          }
        },
        "0ebe50c7f7404ef0926e461dc673b1c0": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8188934c142d4862aaf6559a5e1f60e2",
            "placeholder": "​",
            "style": "IPY_MODEL_25fbd835426d4ebab9eef02ed5cebc69",
            "value": "100%"
          }
        },
        "0ff7c562d0fe412595ffa3badb275e71": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "14d3b49b8dd542d298fef799a774ba91": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "15878b0ec9a247aca534f81378ff5fca": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_184ffa21537a459789826d1348552f76",
            "placeholder": "​",
            "style": "IPY_MODEL_54171321fd0c470f84f1fe58eaccfa6a",
            "value": " 144/144 [00:06&lt;00:00, 20.08it/s]"
          }
        },
        "1620fe07e0c14088bc03ec51f6383846": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "184ffa21537a459789826d1348552f76": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1af9fa979cae41d4b95b3607e4969f80": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1b57fa43b71a4aba9a1c48b32e090a21": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1d7716c911ef47caa5b1875234fb9ed8": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1d9762ab63cd4bdf8a40c92c0a3d17a4": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_71944864009f4728a831eb00192bcc24",
              "IPY_MODEL_9cdfb071b7924b1b92c45bbc3b77fdab",
              "IPY_MODEL_f031bcd482304cf9bbe669a1a1a92ff6"
            ],
            "layout": "IPY_MODEL_b151ec6b8b9543ee9fda04de3da221b5"
          }
        },
        "1ded46589d814836a88da906225cda46": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_64387ba7a3b04f5f8d844ee43146212d",
            "placeholder": "​",
            "style": "IPY_MODEL_3c84b4575d8d45c1866f771bb081aeb4",
            "value": " 144/144 [00:05&lt;00:00, 30.63it/s]"
          }
        },
        "1e837c99ea954afea4ad0fb78656c437": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "1e85072a9aaa4fea99a0f6ea760901dd": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "21ec4ae4727f4308868d2922abae7dc7": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1af9fa979cae41d4b95b3607e4969f80",
            "max": 180,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_da24d7c9b66f42d7a5b0a2abff104d68",
            "value": 180
          }
        },
        "22a664b1fd2a4f9fbfd24f1081746fc0": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "234abcfad26c4c04a5b66f76a7700401": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": "center",
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": "flex",
            "flex": null,
            "flex_flow": "column",
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "25fbd835426d4ebab9eef02ed5cebc69": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "28e8be6b8d7245c395afa2b32f21470c": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2979a00ad822456d80251bbc65086580": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_0ebe50c7f7404ef0926e461dc673b1c0",
              "IPY_MODEL_d11dbd9d6497460580184f1b0050645e",
              "IPY_MODEL_15878b0ec9a247aca534f81378ff5fca"
            ],
            "layout": "IPY_MODEL_7ac670a352d44e51b9a3446b219edb21"
          }
        },
        "2b4ed578eab0412cab5191a1cff83364": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_96ad2b63efe349f2ab068599a0d8c74e",
              "IPY_MODEL_90ca171bbc754e089e502b53713e1188",
              "IPY_MODEL_50c94f552e0f4d90886bc24be7b1e198"
            ],
            "layout": "IPY_MODEL_df9ccf7fb6b149b6b13a81116f0623e9"
          }
        },
        "2c3112527ebd4790b5c9bc7fb84ff494": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6cf92f0162f946049b8d67f7a89dfc27",
            "placeholder": "​",
            "style": "IPY_MODEL_03d73123d0644c8c81d7a0b3f810b8da",
            "value": " 144/144 [00:05&lt;00:00, 34.23it/s]"
          }
        },
        "2f43dfcb6a6b46a0a36406767d040503": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2fbb8019e37846dab7d6445f91645506": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "30bd8772a8ca42a59c6db93865c61581": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "325651b3a4a3456396c773dacb8b693f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "33519b6e3e8b497ebb22ca138ad08c0a": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3407bb38c9c34c08a93eb3f95733b525": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3674a319ee694b939cd4ddda552948d4": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7fefca14bcba4b5092ab98577a088b9f",
            "placeholder": "​",
            "style": "IPY_MODEL_d466487165f84d89bf112aefb990e880",
            "value": "100%"
          }
        },
        "375136b0c3754f79825f46860f010a55": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "388d5aa11c7448ab82effc14c083dace": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "391dfa794c3941c0a608ac7d7353b494": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3ba2350417e54744b9ae1c5567362202": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3c84b4575d8d45c1866f771bb081aeb4": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3dcc933a7d134c39ace2c20beee3028c": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ec4883abcba84051ae9d6c41ddefbcb6",
            "placeholder": "​",
            "style": "IPY_MODEL_a0e05b2d05ba4674983ec9349f57b8ca",
            "value": " 180/180 [00:07&lt;00:00, 28.93it/s]"
          }
        },
        "3e92516b195d413987ec326233626054": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "46e0b6d1197741ee812720e6b599b6c8": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_c8310b424eb941f6ae389d7de87b7287",
              "IPY_MODEL_8fb1bff2394a4aa492a717a0e7db3d2b",
              "IPY_MODEL_74ee415f994d4acf8d31af2e81a93123"
            ],
            "layout": "IPY_MODEL_044171c855c240ff923b69b4e27e1538"
          }
        },
        "480fb7bee6a341c4bf0234aa175bd5e8": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4821854623904c838635611c3725f278": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4bf78c63b25f45dda35014e3b5f6fbce": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_325651b3a4a3456396c773dacb8b693f",
            "placeholder": "​",
            "style": "IPY_MODEL_7fbcaecd08c24b0c895863ca04bb823e",
            "value": " 180/180 [00:06&lt;00:00, 21.78it/s]"
          }
        },
        "4d2ede20c4484505a9d67a9d9300ef15": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4ecd7f1f2b634facbb9b2e48c31295ed": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "508c777cf4ba400388cabd026ed1a08c": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "50c94f552e0f4d90886bc24be7b1e198": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_82a9834fb9bc44199e454e8b92bbcdee",
            "placeholder": "​",
            "style": "IPY_MODEL_2fbb8019e37846dab7d6445f91645506",
            "value": " 2160/2160 [01:26&lt;00:00, 22.69it/s]"
          }
        },
        "51efc4c80b20442ea75c68a7f2afbaed": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "52ef2d85092f4e38aa2af174f41c233a": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "54171321fd0c470f84f1fe58eaccfa6a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5492c6b666a84d59bd64c4b6cba9108f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5530412cffde446ba35ae281cbebf6c2": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "576a2c6d603d4c0486684a47c115f6c3": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "577743f69ee944c6b8efdad9fdde6627": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_0953726feb674665894bddd714143d65",
              "IPY_MODEL_7f122c7354974bb9a6eea53582c0f12d",
              "IPY_MODEL_bd1c647c971a474291feac1b2d87a778"
            ],
            "layout": "IPY_MODEL_06227bde4fac46bbb093003e74953cad"
          }
        },
        "5a6db6d2bd9c4d5697041d3b6cfc2ed6": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5b4e510574df4e6dae68e7cbeda49029",
            "max": 144,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c0047268b94745a8b8010d06f9315199",
            "value": 144
          }
        },
        "5af261b7e4754241abd86e41c98f2750": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5b29a1117851485e9333114fabb4e072": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5b4e510574df4e6dae68e7cbeda49029": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5ec1577140e64c34be0d2860ed137766": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "CheckboxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "CheckboxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "CheckboxView",
            "description": "Add token as git credential?",
            "description_tooltip": null,
            "disabled": false,
            "indent": true,
            "layout": "IPY_MODEL_1d7716c911ef47caa5b1875234fb9ed8",
            "style": "IPY_MODEL_c2593312b66d49bd9119824ba09aa2be",
            "value": true
          }
        },
        "60ec18de81af433880b6167a01fba30f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "621c519f5bbe477ab1d6ba8f740cc382": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6355e7df26a141b183bdfb7f8be79778": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8501c54f82c84882a8705833798ecfb4",
            "placeholder": "​",
            "style": "IPY_MODEL_e39919bc2f864f57abc72184365887ec",
            "value": "100%"
          }
        },
        "63bd8043677442e696ccc0d8a3fe5839": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_508c777cf4ba400388cabd026ed1a08c",
            "placeholder": "​",
            "style": "IPY_MODEL_30bd8772a8ca42a59c6db93865c61581",
            "value": " 180/180 [00:07&lt;00:00, 29.38it/s]"
          }
        },
        "64387ba7a3b04f5f8d844ee43146212d": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "64e799df39eb44e59b36212d393ba581": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_7a4e3ed388964a4da909c7210ead5b9a",
              "IPY_MODEL_00cb5a0118bb438fb38ca3bd304a3016",
              "IPY_MODEL_6ce55cdccad141e5a380f4eb07e96033"
            ],
            "layout": "IPY_MODEL_077b84575f3c4ca5955980a81949216b"
          }
        },
        "65070d65c6bd4e47ae25410c379b0e2b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "6541cdb41ba34ae1a8fdf6d42bbe61d8": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "654bc4b795c94ef280027a0207c5e14f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "65917c03b217485bac9bdd53e962def8": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "65ca015c87154fdb86e63727041b946c": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "66ff90fae10642eca0fa5e631447975b": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6802a79e197b4396b127eb6fb9ea29e7": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_8c389ea18e7c435a801c8f11fa508cf0",
              "IPY_MODEL_b5643a5bb9c54a0983f5b0aefb19d050",
              "IPY_MODEL_bd9df13e01aa4795bb3a7e644dde2bd1"
            ],
            "layout": "IPY_MODEL_4ecd7f1f2b634facbb9b2e48c31295ed"
          }
        },
        "6ce55cdccad141e5a380f4eb07e96033": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ec8361b504df49e7a1ffb60ece4355fd",
            "placeholder": "​",
            "style": "IPY_MODEL_391dfa794c3941c0a608ac7d7353b494",
            "value": " 144/144 [00:05&lt;00:00, 30.34it/s]"
          }
        },
        "6cf92f0162f946049b8d67f7a89dfc27": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6e8904a58f3a4ce3856699fb7000d725": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "70259059c97e4238b778cb1fb53f29e1": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_f5d17117789a44bd8e16584a2f939115",
              "IPY_MODEL_78024e1dc6434eb58caccc64d36d82c3",
              "IPY_MODEL_0bf040751e6841a79713aa8862ca5224"
            ],
            "layout": "IPY_MODEL_654bc4b795c94ef280027a0207c5e14f"
          }
        },
        "70f4de7a068b43dc80053d7508ae9cf4": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "71944864009f4728a831eb00192bcc24": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_28e8be6b8d7245c395afa2b32f21470c",
            "placeholder": "​",
            "style": "IPY_MODEL_5492c6b666a84d59bd64c4b6cba9108f",
            "value": "100%"
          }
        },
        "74ee415f994d4acf8d31af2e81a93123": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7d3c726ca3fb48de98432b35c8550c0b",
            "placeholder": "​",
            "style": "IPY_MODEL_5af261b7e4754241abd86e41c98f2750",
            "value": " 180/180 [00:06&lt;00:00, 29.15it/s]"
          }
        },
        "75995faef432410299c34717b1da96e2": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_9b1f31b34c9342fc8a1b2bfef0cf1f29",
              "IPY_MODEL_81e526b016b7440abf420cc44f64b236",
              "IPY_MODEL_3dcc933a7d134c39ace2c20beee3028c"
            ],
            "layout": "IPY_MODEL_9c78081eb3fe4146a1d79df360cd66eb"
          }
        },
        "7793574382e94f4fb1eb22c287bcfb02": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "78024e1dc6434eb58caccc64d36d82c3": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_05e002ce5986414c816e4435c631404b",
            "max": 144,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_bfcf9873f5f14d559f3f0906d6d4b62d",
            "value": 144
          }
        },
        "78715c8355a84e7a950ee5e0541d598f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_ed976a6e6c9f40b2810d7869459711d4",
              "IPY_MODEL_21ec4ae4727f4308868d2922abae7dc7",
              "IPY_MODEL_4bf78c63b25f45dda35014e3b5f6fbce"
            ],
            "layout": "IPY_MODEL_84bbf960ddee46e39a41007929a386f1"
          }
        },
        "7a4e3ed388964a4da909c7210ead5b9a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_52ef2d85092f4e38aa2af174f41c233a",
            "placeholder": "​",
            "style": "IPY_MODEL_a6229b8c298947498453f64d0e90d004",
            "value": "100%"
          }
        },
        "7ac670a352d44e51b9a3446b219edb21": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7d3c726ca3fb48de98432b35c8550c0b": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7f122c7354974bb9a6eea53582c0f12d": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9497f8b2ca644fa981a0acf97fcb8ef8",
            "max": 144,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_aae2ea570bca4227952e0cf9e9b39243",
            "value": 144
          }
        },
        "7f924ae9b05f451b9a6c8eb4920f2c2b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_addabcbe6ef94a49b1ab76027ce7bd31",
            "max": 180,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_1e837c99ea954afea4ad0fb78656c437",
            "value": 180
          }
        },
        "7fbcaecd08c24b0c895863ca04bb823e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7fefca14bcba4b5092ab98577a088b9f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8109a631fa3f4443885ecd6953c1137e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "811ebb000e664308b9a9dc519a70deab": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8188934c142d4862aaf6559a5e1f60e2": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "81e526b016b7440abf420cc44f64b236": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1b57fa43b71a4aba9a1c48b32e090a21",
            "max": 180,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_375136b0c3754f79825f46860f010a55",
            "value": 180
          }
        },
        "825fb3ad8d7c408f97c2a2939ca0640d": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_b012ced6282544ffa601fcdf819930c7",
              "IPY_MODEL_91d87964d9a74543b9c95c80f3371187",
              "IPY_MODEL_e98f8f6751114942bbd8bf8d89366e41"
            ],
            "layout": "IPY_MODEL_22a664b1fd2a4f9fbfd24f1081746fc0"
          }
        },
        "82a9834fb9bc44199e454e8b92bbcdee": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "830fda49b567472b91d0c2c210db34cf": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_8d3b5565501247f7954ae0b18dba7c58",
              "IPY_MODEL_7f924ae9b05f451b9a6c8eb4920f2c2b",
              "IPY_MODEL_63bd8043677442e696ccc0d8a3fe5839"
            ],
            "layout": "IPY_MODEL_91970688d85e42eb90aa86ccd0a91be7"
          }
        },
        "84bbf960ddee46e39a41007929a386f1": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8501c54f82c84882a8705833798ecfb4": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "886acd0422994886888136cc71f23519": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "88861a3614834f7fb3bdc99de0d2baeb": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "8c04505825f449bf8669c1ad29ee706c": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8c21b9e91a28441f8297d3a04f5b4d9a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "8c389ea18e7c435a801c8f11fa508cf0": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ff227884d1644848b0185b93c0fa6868",
            "placeholder": "​",
            "style": "IPY_MODEL_07cbbe5fb260457987b2464bb84268fe",
            "value": "100%"
          }
        },
        "8ca69d3bc7d5458288f648a3764b3a84": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "8d0ec28c53b042f78568e8d2323e4d29": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3407bb38c9c34c08a93eb3f95733b525",
            "max": 180,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_ec0581dd1df844639b90b53a455454f9",
            "value": 180
          }
        },
        "8d3b5565501247f7954ae0b18dba7c58": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cdffa3c04b2a4318adac0c9070e40200",
            "placeholder": "​",
            "style": "IPY_MODEL_8109a631fa3f4443885ecd6953c1137e",
            "value": "100%"
          }
        },
        "8e7c46f6e6624ea8926d067813711c89": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8fb1bff2394a4aa492a717a0e7db3d2b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_66ff90fae10642eca0fa5e631447975b",
            "max": 180,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_88861a3614834f7fb3bdc99de0d2baeb",
            "value": 180
          }
        },
        "9025a4ebdb5f4881aaf62e5f84e6414d": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_3674a319ee694b939cd4ddda552948d4",
              "IPY_MODEL_8d0ec28c53b042f78568e8d2323e4d29",
              "IPY_MODEL_0d12e07a6cd747588e0d8360a4d9a8d5"
            ],
            "layout": "IPY_MODEL_8e7c46f6e6624ea8926d067813711c89"
          }
        },
        "90ca171bbc754e089e502b53713e1188": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_576a2c6d603d4c0486684a47c115f6c3",
            "max": 2160,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_eaa96b03fde14c8298310984e35dfda6",
            "value": 2160
          }
        },
        "91970688d85e42eb90aa86ccd0a91be7": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "91d87964d9a74543b9c95c80f3371187": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_51efc4c80b20442ea75c68a7f2afbaed",
            "max": 144,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_8ca69d3bc7d5458288f648a3764b3a84",
            "value": 144
          }
        },
        "9408900617b942e394ced6a0d8b4d799": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ButtonModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ButtonModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ButtonView",
            "button_style": "",
            "description": "Login",
            "disabled": false,
            "icon": "",
            "layout": "IPY_MODEL_ce5debeb4ec54287a2cc248cdb637651",
            "style": "IPY_MODEL_b21edc97e6b04b3ea5eee2aae3f4eb86",
            "tooltip": ""
          }
        },
        "9497f8b2ca644fa981a0acf97fcb8ef8": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "96ad2b63efe349f2ab068599a0d8c74e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_063c166bc4b44f0292ca6ff9133cd0b7",
            "placeholder": "​",
            "style": "IPY_MODEL_fffaff2fb61446a5b900e35345b2b5eb",
            "value": "100%"
          }
        },
        "97e98f74d3b74e38be50fb205d6f1efd": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9b1f31b34c9342fc8a1b2bfef0cf1f29": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0abe281fc6e5443e87a9a7d28419783a",
            "placeholder": "​",
            "style": "IPY_MODEL_aeb95b87161f46f6bf173943b77a4e52",
            "value": "100%"
          }
        },
        "9c78081eb3fe4146a1d79df360cd66eb": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9cdfb071b7924b1b92c45bbc3b77fdab": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8c04505825f449bf8669c1ad29ee706c",
            "max": 180,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_8c21b9e91a28441f8297d3a04f5b4d9a",
            "value": 180
          }
        },
        "a0e05b2d05ba4674983ec9349f57b8ca": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "a6229b8c298947498453f64d0e90d004": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "aae2ea570bca4227952e0cf9e9b39243": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "addabcbe6ef94a49b1ab76027ce7bd31": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "aeb95b87161f46f6bf173943b77a4e52": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b012ced6282544ffa601fcdf819930c7": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_65ca015c87154fdb86e63727041b946c",
            "placeholder": "​",
            "style": "IPY_MODEL_811ebb000e664308b9a9dc519a70deab",
            "value": "100%"
          }
        },
        "b08c648d67474d9eb9986ea24f28311b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d7754c6b0a7a4ba6b40b3c1ffc458b55",
            "max": 144,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_f0f946dd323d47f88ed986c4c0a2537f",
            "value": 144
          }
        },
        "b151ec6b8b9543ee9fda04de3da221b5": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b21edc97e6b04b3ea5eee2aae3f4eb86": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ButtonStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ButtonStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "button_color": null,
            "font_weight": ""
          }
        },
        "b2b7a69c70d7489b8dd9cd4475fb22f9": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b5643a5bb9c54a0983f5b0aefb19d050": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4d2ede20c4484505a9d67a9d9300ef15",
            "max": 144,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_1620fe07e0c14088bc03ec51f6383846",
            "value": 144
          }
        },
        "b7021c7fad1643368f77f69290c1b5ae": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b76c6442906a46f0b164449244f518eb": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "VBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "VBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "VBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_b95e718ee1a0448aa9150505e439fad3",
              "IPY_MODEL_bb7569ae4d9e4996a7c74e6d6b0e172e",
              "IPY_MODEL_5ec1577140e64c34be0d2860ed137766",
              "IPY_MODEL_9408900617b942e394ced6a0d8b4d799",
              "IPY_MODEL_d3aeb6e3ae164e33812fec42f2205e08"
            ],
            "layout": "IPY_MODEL_234abcfad26c4c04a5b66f76a7700401"
          }
        },
        "b95e718ee1a0448aa9150505e439fad3": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7793574382e94f4fb1eb22c287bcfb02",
            "placeholder": "​",
            "style": "IPY_MODEL_6541cdb41ba34ae1a8fdf6d42bbe61d8",
            "value": "<center> <img\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.svg\nalt='Hugging Face'> <br> Copy a token from <a\nhref=\"https://huggingface.co/settings/tokens\" target=\"_blank\">your Hugging Face\ntokens page</a> and paste it below. <br> Immediately click login after copying\nyour token or it might be stored in plain text in this notebook file. </center>"
          }
        },
        "b9c8ade472944666a4cc4f6793ec0f35": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bb40ed2d47ed4e3cbe89e68e816677e4": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_0c1eb5b72748416ba1a78a06543a7bb8",
              "IPY_MODEL_b08c648d67474d9eb9986ea24f28311b",
              "IPY_MODEL_1ded46589d814836a88da906225cda46"
            ],
            "layout": "IPY_MODEL_2f43dfcb6a6b46a0a36406767d040503"
          }
        },
        "bb7569ae4d9e4996a7c74e6d6b0e172e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "PasswordModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "PasswordModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "PasswordView",
            "continuous_update": true,
            "description": "Token:",
            "description_tooltip": null,
            "disabled": false,
            "layout": "IPY_MODEL_5b29a1117851485e9333114fabb4e072",
            "placeholder": "​",
            "style": "IPY_MODEL_bc8a2cda603a424299c53080b6050273",
            "value": ""
          }
        },
        "bc8a2cda603a424299c53080b6050273": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "bd1c647c971a474291feac1b2d87a778": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b2b7a69c70d7489b8dd9cd4475fb22f9",
            "placeholder": "​",
            "style": "IPY_MODEL_621c519f5bbe477ab1d6ba8f740cc382",
            "value": " 144/144 [00:04&lt;00:00, 28.77it/s]"
          }
        },
        "bd9df13e01aa4795bb3a7e644dde2bd1": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_886acd0422994886888136cc71f23519",
            "placeholder": "​",
            "style": "IPY_MODEL_14d3b49b8dd542d298fef799a774ba91",
            "value": " 144/144 [00:05&lt;00:00, 30.99it/s]"
          }
        },
        "bfcf9873f5f14d559f3f0906d6d4b62d": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "c0047268b94745a8b8010d06f9315199": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "c0d8e052762f4742a4589527fced5fd2": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_6355e7df26a141b183bdfb7f8be79778",
              "IPY_MODEL_5a6db6d2bd9c4d5697041d3b6cfc2ed6",
              "IPY_MODEL_2c3112527ebd4790b5c9bc7fb84ff494"
            ],
            "layout": "IPY_MODEL_ce028da02fad43608ca2d68936a66129"
          }
        },
        "c2593312b66d49bd9119824ba09aa2be": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c8310b424eb941f6ae389d7de87b7287": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_65917c03b217485bac9bdd53e962def8",
            "placeholder": "​",
            "style": "IPY_MODEL_60ec18de81af433880b6167a01fba30f",
            "value": "100%"
          }
        },
        "cda06f33eb604e25aea0c0b09d47827e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "cdffa3c04b2a4318adac0c9070e40200": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ce028da02fad43608ca2d68936a66129": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ce5debeb4ec54287a2cc248cdb637651": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d11dbd9d6497460580184f1b0050645e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3e92516b195d413987ec326233626054",
            "max": 144,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_cda06f33eb604e25aea0c0b09d47827e",
            "value": 144
          }
        },
        "d3aeb6e3ae164e33812fec42f2205e08": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fb65972194544c41b8977d99eaa97718",
            "placeholder": "​",
            "style": "IPY_MODEL_97e98f74d3b74e38be50fb205d6f1efd",
            "value": "\n<b>Pro Tip:</b> If you don't already have one, you can create a dedicated\n'notebooks' token with 'write' access, that you can then easily reuse for all\nnotebooks. </center>"
          }
        },
        "d466487165f84d89bf112aefb990e880": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d4f67196e74e4ff9b5df7045ed98b17a": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d7754c6b0a7a4ba6b40b3c1ffc458b55": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "da24d7c9b66f42d7a5b0a2abff104d68": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "db81ac65ab5847819ccf4c083d049302": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "df27d8e8836146c7a5ff13850032924f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "df9ccf7fb6b149b6b13a81116f0623e9": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e39919bc2f864f57abc72184365887ec": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e98f8f6751114942bbd8bf8d89366e41": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_df27d8e8836146c7a5ff13850032924f",
            "placeholder": "​",
            "style": "IPY_MODEL_388d5aa11c7448ab82effc14c083dace",
            "value": " 144/144 [00:04&lt;00:00, 29.19it/s]"
          }
        },
        "eaa96b03fde14c8298310984e35dfda6": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "ec0581dd1df844639b90b53a455454f9": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "ec4883abcba84051ae9d6c41ddefbcb6": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ec8361b504df49e7a1ffb60ece4355fd": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ed976a6e6c9f40b2810d7869459711d4": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6e8904a58f3a4ce3856699fb7000d725",
            "placeholder": "​",
            "style": "IPY_MODEL_480fb7bee6a341c4bf0234aa175bd5e8",
            "value": "100%"
          }
        },
        "f031bcd482304cf9bbe669a1a1a92ff6": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_db81ac65ab5847819ccf4c083d049302",
            "placeholder": "​",
            "style": "IPY_MODEL_1e85072a9aaa4fea99a0f6ea760901dd",
            "value": " 180/180 [00:06&lt;00:00, 27.93it/s]"
          }
        },
        "f0f946dd323d47f88ed986c4c0a2537f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "f5947be879d44715846b197614974298": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f5d17117789a44bd8e16584a2f939115": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_33519b6e3e8b497ebb22ca138ad08c0a",
            "placeholder": "​",
            "style": "IPY_MODEL_0ff7c562d0fe412595ffa3badb275e71",
            "value": "100%"
          }
        },
        "f80833a8b8f442409dec918c4aba25c1": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "fb65972194544c41b8977d99eaa97718": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ff227884d1644848b0185b93c0fa6868": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fffaff2fb61446a5b900e35345b2b5eb": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}