tutorials/tutorial-spelling-correction.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "4e7d0630b400492996bbfc64fdb64472": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_7d1d5591ed3d4e289b258f1ee0a87591",
              "IPY_MODEL_5ec88ae212e2434e9b8df37fd06d27e6",
              "IPY_MODEL_b069d5056aed4b12b5781ecde3b09ad3"
            ],
            "layout": "IPY_MODEL_73926514d6cc4fd4889c776e3df168ab"
          }
        },
        "7d1d5591ed3d4e289b258f1ee0a87591": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2b35fc599bfc47948bef702a0ad172b0",
            "placeholder": "​",
            "style": "IPY_MODEL_05a98a12bbbe4b2e98e81afa6cbc30d0",
            "value": "pt.Experiment: 100%"
          }
        },
        "5ec88ae212e2434e9b8df37fd06d27e6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_431b8ccf27aa4226b8822ee415f6b8cb",
            "max": 10,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_dfba6879893d4b16a1792afb819b3ff2",
            "value": 10
          }
        },
        "b069d5056aed4b12b5781ecde3b09ad3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6c18f9f2716248e6a614318ca8399d56",
            "placeholder": "​",
            "style": "IPY_MODEL_42fc256c1eae47fe9977b1743f962afb",
            "value": " 10/10 [01:19&lt;00:00,  7.54s/system]"
          }
        },
        "73926514d6cc4fd4889c776e3df168ab": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2b35fc599bfc47948bef702a0ad172b0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "05a98a12bbbe4b2e98e81afa6cbc30d0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "431b8ccf27aa4226b8822ee415f6b8cb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "dfba6879893d4b16a1792afb819b3ff2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "6c18f9f2716248e6a614318ca8399d56": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "42fc256c1eae47fe9977b1743f962afb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b86ef2f2bb804898aef52b026f4fe8f3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_b445e1dbd2b74592bd4e93bfc976eade",
              "IPY_MODEL_63d409426256491a945aa6e9fadc91e0",
              "IPY_MODEL_093d08d73e614dc28c45610edb97428b"
            ],
            "layout": "IPY_MODEL_62e8c529708f4579878331d3aa5fdb5c"
          }
        },
        "b445e1dbd2b74592bd4e93bfc976eade": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_83197e1f780c4257a39531f194825d46",
            "placeholder": "​",
            "style": "IPY_MODEL_237e08f82d724184b1dcb9438044c7f8",
            "value": "pt.Experiment: 100%"
          }
        },
        "63d409426256491a945aa6e9fadc91e0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e037a00aeea44f53b39f8c5af8c38dc7",
            "max": 5,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_fcafdfb4ddda44ad9f2f76ce0ad31d8e",
            "value": 5
          }
        },
        "093d08d73e614dc28c45610edb97428b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0b355dc157164f8fa5dff8c36620e7f1",
            "placeholder": "​",
            "style": "IPY_MODEL_cb46c352c0df46a594345963d7dc9e77",
            "value": " 5/5 [00:35&lt;00:00,  6.88s/system]"
          }
        },
        "62e8c529708f4579878331d3aa5fdb5c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "83197e1f780c4257a39531f194825d46": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "237e08f82d724184b1dcb9438044c7f8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e037a00aeea44f53b39f8c5af8c38dc7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fcafdfb4ddda44ad9f2f76ce0ad31d8e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "0b355dc157164f8fa5dff8c36620e7f1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cb46c352c0df46a594345963d7dc9e77": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "fddd7a8e2e214cafb02c1e49634074fe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_c2c8c098371f41e7a56a054a809e482d",
              "IPY_MODEL_9a81772c00c24cf2aff7d6bc6255a514",
              "IPY_MODEL_8590ae86846f4123969a12f752db99ef"
            ],
            "layout": "IPY_MODEL_1292f41a5606410399670c3a24882e2b"
          }
        },
        "c2c8c098371f41e7a56a054a809e482d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_64733394b3314329b03b5220f90cf0ba",
            "placeholder": "​",
            "style": "IPY_MODEL_e0d55e4c78dc4e238c978530d4f43a50",
            "value": "pt.Experiment: 100%"
          }
        },
        "9a81772c00c24cf2aff7d6bc6255a514": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c9f95b527e1b4eeba4cd06a04880685b",
            "max": 5,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_1305ae826bd44e28a0c3b1d92aba931b",
            "value": 5
          }
        },
        "8590ae86846f4123969a12f752db99ef": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fbf6d5435b8f46f7831b10e793d5c658",
            "placeholder": "​",
            "style": "IPY_MODEL_99984cc2bbbd40aeaef689e6b005df10",
            "value": " 5/5 [00:36&lt;00:00,  7.17s/system]"
          }
        },
        "1292f41a5606410399670c3a24882e2b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "64733394b3314329b03b5220f90cf0ba": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e0d55e4c78dc4e238c978530d4f43a50": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c9f95b527e1b4eeba4cd06a04880685b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1305ae826bd44e28a0c3b1d92aba931b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "fbf6d5435b8f46f7831b10e793d5c658": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "99984cc2bbbd40aeaef689e6b005df10": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8bfe12cbd0a24e0bb317792e8131881e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_6c24a096499b4fd7ac5c396552b2114b",
              "IPY_MODEL_901b4a15631e4ec1a92ff401054c2a29",
              "IPY_MODEL_dbc2241bcbc74d3e9bc76cd7c4ca859d"
            ],
            "layout": "IPY_MODEL_dc59551001e042699fab159425307c21"
          }
        },
        "6c24a096499b4fd7ac5c396552b2114b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7226dcc59ee4499f9c703ee6d4a1e76d",
            "placeholder": "​",
            "style": "IPY_MODEL_6348acf11cd346b3a88af185789cd17a",
            "value": "pt.Experiment: 100%"
          }
        },
        "901b4a15631e4ec1a92ff401054c2a29": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_596540bb73f942eb983764762666ed7f",
            "max": 10,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_0a0a4b5cd8444a6290a9629b17c0191a",
            "value": 10
          }
        },
        "dbc2241bcbc74d3e9bc76cd7c4ca859d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_84d401a4455c4951aed9a8589276cdd9",
            "placeholder": "​",
            "style": "IPY_MODEL_f920fa10e3404730bf804f1e32957a45",
            "value": " 10/10 [00:58&lt;00:00,  5.91s/system]"
          }
        },
        "dc59551001e042699fab159425307c21": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7226dcc59ee4499f9c703ee6d4a1e76d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6348acf11cd346b3a88af185789cd17a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "596540bb73f942eb983764762666ed7f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0a0a4b5cd8444a6290a9629b17c0191a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "84d401a4455c4951aed9a8589276cdd9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f920fa10e3404730bf804f1e32957a45": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "dbde6585132a44c9a0fd36e9983d5437": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_0901902d78ca4bcdb565973e9481f43b",
              "IPY_MODEL_09deff9aff8149358dacb8ee9988b587",
              "IPY_MODEL_15908a55914644e7a68e840cd5b36583"
            ],
            "layout": "IPY_MODEL_67f986cf6bb24ceaaa39ddd3dbf6c6c0"
          }
        },
        "0901902d78ca4bcdb565973e9481f43b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a489ea48509e421183a31c80587dac25",
            "placeholder": "​",
            "style": "IPY_MODEL_7e825ae484664cc9807bd1bce32190d6",
            "value": "pt.Experiment: 100%"
          }
        },
        "09deff9aff8149358dacb8ee9988b587": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ed98202dafc640b98ddc00ed18ec2871",
            "max": 5,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_4549dbd08e4c45f4b6811d56679f5e90",
            "value": 5
          }
        },
        "15908a55914644e7a68e840cd5b36583": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0253cbce570a44f4b60bee1b71a4b98a",
            "placeholder": "​",
            "style": "IPY_MODEL_9f4b0fde314f4b9096fdd678e4c8bb43",
            "value": " 5/5 [00:25&lt;00:00,  5.24s/system]"
          }
        },
        "67f986cf6bb24ceaaa39ddd3dbf6c6c0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a489ea48509e421183a31c80587dac25": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7e825ae484664cc9807bd1bce32190d6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ed98202dafc640b98ddc00ed18ec2871": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4549dbd08e4c45f4b6811d56679f5e90": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "0253cbce570a44f4b60bee1b71a4b98a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9f4b0fde314f4b9096fdd678e4c8bb43": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "cd92d09025e249f698e35bc7bb573ba1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_97c55f41783d4f0aae80a1b25bf19720",
              "IPY_MODEL_0c5d7a44599d4336801961063d3b42d8",
              "IPY_MODEL_0600fa99f9a6486dac4b44ca717455ff"
            ],
            "layout": "IPY_MODEL_f117c5c455c248d78f63034fc1162254"
          }
        },
        "97c55f41783d4f0aae80a1b25bf19720": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f3328d097a0b47e981a45cb0dd19d70b",
            "placeholder": "​",
            "style": "IPY_MODEL_6542c8713aa14a9e8883e4cfb59108e5",
            "value": "pt.Experiment: 100%"
          }
        },
        "0c5d7a44599d4336801961063d3b42d8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_83b8e6d8a4494c328bd7138816dd2c29",
            "max": 5,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_4c22749ff701428ebf2ca824871010c5",
            "value": 5
          }
        },
        "0600fa99f9a6486dac4b44ca717455ff": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e300abbf67aa408fb8314a97b487655a",
            "placeholder": "​",
            "style": "IPY_MODEL_544dfb9e84854160a9d45f83ebbf598f",
            "value": " 5/5 [00:20&lt;00:00,  4.06s/system]"
          }
        },
        "f117c5c455c248d78f63034fc1162254": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f3328d097a0b47e981a45cb0dd19d70b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6542c8713aa14a9e8883e4cfb59108e5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "83b8e6d8a4494c328bd7138816dd2c29": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4c22749ff701428ebf2ca824871010c5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "e300abbf67aa408fb8314a97b487655a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "544dfb9e84854160a9d45f83ebbf598f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "a1d1ebaeff0247279f721274a13dd0ae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_2fc1f39b71294399aa512e98247ea7a2",
              "IPY_MODEL_eacf9e7b6641464a9e1f3364b20c6cd1",
              "IPY_MODEL_fb33580cf6cc48c2a2f1b19142892baa"
            ],
            "layout": "IPY_MODEL_21c71ecb34f142cb8440f2af712189a6"
          }
        },
        "2fc1f39b71294399aa512e98247ea7a2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_49b4ed4ec1eb4d9cbf425c562994638d",
            "placeholder": "​",
            "style": "IPY_MODEL_5c3b573dfa314012bc278d6c68e054b4",
            "value": "pt.Experiment: 100%"
          }
        },
        "eacf9e7b6641464a9e1f3364b20c6cd1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_90d17596f92849b7b65eb5eb4ec6f3b3",
            "max": 10,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_2e6ba146380b4b318383a717c5f1dad4",
            "value": 10
          }
        },
        "fb33580cf6cc48c2a2f1b19142892baa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_73df9e84e1934150b02bb4743e6ae2fb",
            "placeholder": "​",
            "style": "IPY_MODEL_37f14440d16d4f63a5997f9f0b345367",
            "value": " 10/10 [00:45&lt;00:00,  4.62s/system]"
          }
        },
        "21c71ecb34f142cb8440f2af712189a6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "49b4ed4ec1eb4d9cbf425c562994638d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5c3b573dfa314012bc278d6c68e054b4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "90d17596f92849b7b65eb5eb4ec6f3b3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2e6ba146380b4b318383a717c5f1dad4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "73df9e84e1934150b02bb4743e6ae2fb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "37f14440d16d4f63a5997f9f0b345367": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "fae8aaab07f04bce9a601549a3124bc6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_ec4b89232da94970a1f7288b1e29afa2",
              "IPY_MODEL_486d035e687441578d23e7eb2e47a5aa",
              "IPY_MODEL_c997c5b8cc1842c886b8dda6a6710ef9"
            ],
            "layout": "IPY_MODEL_38dfa4b9d8fb43589e8bcdf75f0dd5b5"
          }
        },
        "ec4b89232da94970a1f7288b1e29afa2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_290f7c66a3fa418eb500b3232226461e",
            "placeholder": "​",
            "style": "IPY_MODEL_dc2f2f0ac9874f48be7038e6bc74dcc1",
            "value": "pt.Experiment: 100%"
          }
        },
        "486d035e687441578d23e7eb2e47a5aa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_575739d54daa4fcd96029274ef29a788",
            "max": 5,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_686430e8f0714318ac4a784f23e17b24",
            "value": 5
          }
        },
        "c997c5b8cc1842c886b8dda6a6710ef9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_90cf03d22a77451383e6ddf7d1069d14",
            "placeholder": "​",
            "style": "IPY_MODEL_74a086355ce44703b61c816d702bd681",
            "value": " 5/5 [00:31&lt;00:00,  6.17s/system]"
          }
        },
        "38dfa4b9d8fb43589e8bcdf75f0dd5b5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "290f7c66a3fa418eb500b3232226461e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "dc2f2f0ac9874f48be7038e6bc74dcc1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "575739d54daa4fcd96029274ef29a788": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "686430e8f0714318ac4a784f23e17b24": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "90cf03d22a77451383e6ddf7d1069d14": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "74a086355ce44703b61c816d702bd681": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b2b5a240658349c891bcddf419df6971": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_4a95a1669d6b42b8a584ff576a06073e",
              "IPY_MODEL_f442063c7e54430bba7ec02c0c2ee54c",
              "IPY_MODEL_8441b160fd5a4b11a49ca4d7e3bbc3df"
            ],
            "layout": "IPY_MODEL_31b88a7fcb0d4fd7bdaea441b8737a19"
          }
        },
        "4a95a1669d6b42b8a584ff576a06073e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7c7859222aa3449e83aeaacb3ecdce19",
            "placeholder": "​",
            "style": "IPY_MODEL_1514f1eb6e4c4f9d8904dc5b1b4c8f07",
            "value": "pt.Experiment: 100%"
          }
        },
        "f442063c7e54430bba7ec02c0c2ee54c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_02f99b0379744100999869fc36cf9e30",
            "max": 5,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_7509d5f93bf64d85946d4ec2516e383e",
            "value": 5
          }
        },
        "8441b160fd5a4b11a49ca4d7e3bbc3df": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d2916746d061427ab6f560f03a8bac47",
            "placeholder": "​",
            "style": "IPY_MODEL_00bb6162e83c44599dbc50d5a1e4a1fc",
            "value": " 5/5 [00:25&lt;00:00,  4.82s/system]"
          }
        },
        "31b88a7fcb0d4fd7bdaea441b8737a19": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7c7859222aa3449e83aeaacb3ecdce19": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1514f1eb6e4c4f9d8904dc5b1b4c8f07": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "02f99b0379744100999869fc36cf9e30": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7509d5f93bf64d85946d4ec2516e383e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "d2916746d061427ab6f560f03a8bac47": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "00bb6162e83c44599dbc50d5a1e4a1fc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "n7wnwr6s581m",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f93d60e4-96fd-401c-c76f-d015f67cf569"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting git+https://github.com/tira-io/tira.git@development#&subdirectory=python-client\n",
            "  Cloning https://github.com/tira-io/tira.git (to revision development) to /tmp/pip-req-build-p2wvdwgz\n",
            "  Running command git clone --filter=blob:none --quiet https://github.com/tira-io/tira.git /tmp/pip-req-build-p2wvdwgz\n",
            "  Running command git checkout -b development --track origin/development\n",
            "  Switched to a new branch 'development'\n",
            "  Branch 'development' set up to track remote branch 'development' from 'origin'.\n",
            "  Resolved https://github.com/tira-io/tira.git to commit e06d51d38ac6b5c95d3bf5735631f5ca34fb1586\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting python-terrier\n",
            "  Downloading python-terrier-0.10.0.tar.gz (107 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m107.6/107.6 kB\u001b[0m \u001b[31m2.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting ir-datasets\n",
            "  Downloading ir_datasets-0.5.6-py3-none-any.whl (335 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m335.2/335.2 kB\u001b[0m \u001b[31m9.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from python-terrier) (1.25.2)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from python-terrier) (1.5.3)\n",
            "Collecting wget (from python-terrier)\n",
            "  Downloading wget-3.2.zip (10 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from python-terrier) (4.66.2)\n",
            "Collecting pyjnius>=1.4.2 (from python-terrier)\n",
            "  Downloading pyjnius-1.6.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.5 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.5/1.5 MB\u001b[0m \u001b[31m13.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting matchpy (from python-terrier)\n",
            "  Downloading matchpy-0.5.5-py3-none-any.whl (69 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m69.6/69.6 kB\u001b[0m \u001b[31m9.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: scikit-learn in /usr/local/lib/python3.10/dist-packages (from python-terrier) (1.2.2)\n",
            "Collecting deprecated (from python-terrier)\n",
            "  Downloading Deprecated-1.2.14-py2.py3-none-any.whl (9.6 kB)\n",
            "Collecting chest (from python-terrier)\n",
            "  Downloading chest-0.2.3.tar.gz (9.6 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from python-terrier) (1.11.4)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from python-terrier) (2.31.0)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from python-terrier) (1.3.2)\n",
            "Collecting nptyping==1.4.4 (from python-terrier)\n",
            "  Downloading nptyping-1.4.4-py3-none-any.whl (31 kB)\n",
            "Requirement already satisfied: more_itertools in /usr/local/lib/python3.10/dist-packages (from python-terrier) (10.1.0)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from python-terrier) (3.1.3)\n",
            "Requirement already satisfied: statsmodels in /usr/local/lib/python3.10/dist-packages (from python-terrier) (0.14.1)\n",
            "Collecting ir_measures>=0.3.1 (from python-terrier)\n",
            "  Downloading ir_measures-0.3.3.tar.gz (48 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m48.8/48.8 kB\u001b[0m \u001b[31m6.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting dill (from python-terrier)\n",
            "  Downloading dill-0.3.8-py3-none-any.whl (116 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m116.3/116.3 kB\u001b[0m \u001b[31m15.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pytrec_eval_terrier>=0.5.3 (from python-terrier)\n",
            "  Downloading pytrec_eval_terrier-0.5.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (287 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m287.4/287.4 kB\u001b[0m \u001b[31m13.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting typish>=1.7.0 (from nptyping==1.4.4->python-terrier)\n",
            "  Downloading typish-1.9.3-py3-none-any.whl (45 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m45.1/45.1 kB\u001b[0m \u001b[31m5.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: beautifulsoup4>=4.4.1 in /usr/local/lib/python3.10/dist-packages (from ir-datasets) (4.12.3)\n",
            "Collecting inscriptis>=2.2.0 (from ir-datasets)\n",
            "  Downloading inscriptis-2.4.0.1-py3-none-any.whl (41 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m41.7/41.7 kB\u001b[0m \u001b[31m5.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: lxml>=4.5.2 in /usr/local/lib/python3.10/dist-packages (from ir-datasets) (4.9.4)\n",
            "Requirement already satisfied: pyyaml>=5.3.1 in /usr/local/lib/python3.10/dist-packages (from ir-datasets) (6.0.1)\n",
            "Collecting trec-car-tools>=2.5.4 (from ir-datasets)\n",
            "  Downloading trec_car_tools-2.6-py3-none-any.whl (8.4 kB)\n",
            "Collecting lz4>=3.1.10 (from ir-datasets)\n",
            "  Downloading lz4-4.3.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m17.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting warc3-wet>=0.2.3 (from ir-datasets)\n",
            "  Downloading warc3_wet-0.2.3-py3-none-any.whl (13 kB)\n",
            "Collecting warc3-wet-clueweb09>=0.2.5 (from ir-datasets)\n",
            "  Downloading warc3-wet-clueweb09-0.2.5.tar.gz (17 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting zlib-state>=0.1.3 (from ir-datasets)\n",
            "  Downloading zlib-state-0.1.6.tar.gz (9.5 kB)\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Installing backend dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting ijson>=3.1.3 (from ir-datasets)\n",
            "  Downloading ijson-3.2.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (111 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m111.8/111.8 kB\u001b[0m \u001b[31m11.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pyautocorpus>=0.1.1 (from ir-datasets)\n",
            "  Downloading pyautocorpus-0.1.12-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (379 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m379.9/379.9 kB\u001b[0m \u001b[31m14.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting unlzw3>=0.2.1 (from ir-datasets)\n",
            "  Downloading unlzw3-0.2.2-py3-none-any.whl (6.1 kB)\n",
            "Collecting docker==6.*,>=6.0.0 (from tira==0.0.105)\n",
            "  Downloading docker-6.1.3-py3-none-any.whl (148 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m148.1/148.1 kB\u001b[0m \u001b[31m16.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: packaging>=14.0 in /usr/local/lib/python3.10/dist-packages (from docker==6.*,>=6.0.0->tira==0.0.105) (23.2)\n",
            "Requirement already satisfied: urllib3>=1.26.0 in /usr/local/lib/python3.10/dist-packages (from docker==6.*,>=6.0.0->tira==0.0.105) (2.0.7)\n",
            "Requirement already satisfied: websocket-client>=0.32.0 in /usr/local/lib/python3.10/dist-packages (from docker==6.*,>=6.0.0->tira==0.0.105) (1.7.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->python-terrier) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->python-terrier) (3.6)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->python-terrier) (2024.2.2)\n",
            "Requirement already satisfied: soupsieve>1.2 in /usr/local/lib/python3.10/dist-packages (from beautifulsoup4>=4.4.1->ir-datasets) (2.5)\n",
            "Collecting cwl-eval>=1.0.10 (from ir_measures>=0.3.1->python-terrier)\n",
            "  Downloading cwl-eval-1.0.12.tar.gz (31 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting cbor>=1.0.0 (from trec-car-tools>=2.5.4->ir-datasets)\n",
            "  Downloading cbor-1.0.0.tar.gz (20 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting heapdict (from chest->python-terrier)\n",
            "  Downloading HeapDict-1.0.1-py3-none-any.whl (3.9 kB)\n",
            "Requirement already satisfied: wrapt<2,>=1.10 in /usr/local/lib/python3.10/dist-packages (from deprecated->python-terrier) (1.14.1)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->python-terrier) (2.1.5)\n",
            "Collecting multiset<3.0,>=2.0 (from matchpy->python-terrier)\n",
            "  Downloading multiset-2.1.1-py2.py3-none-any.whl (8.8 kB)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->python-terrier) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->python-terrier) (2023.4)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->python-terrier) (3.3.0)\n",
            "Requirement already satisfied: patsy>=0.5.4 in /usr/local/lib/python3.10/dist-packages (from statsmodels->python-terrier) (0.5.6)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.10/dist-packages (from patsy>=0.5.4->statsmodels->python-terrier) (1.16.0)\n",
            "Building wheels for collected packages: python-terrier, tira, ir_measures, warc3-wet-clueweb09, zlib-state, chest, wget, cbor, cwl-eval\n",
            "  Building wheel for python-terrier (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for python-terrier: filename=python_terrier-0.10.0-py3-none-any.whl size=115532 sha256=f8c486aef1f7ad0fd046721833ab66bdce4f158b8204c71565da5f6bea954b85\n",
            "  Stored in directory: /root/.cache/pip/wheels/79/7c/8f/679a982895c53af35178eceda648a4bc9a9af6af5542e31a0e\n",
            "  Building wheel for tira (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for tira: filename=tira-0.0.105-py3-none-any.whl size=49283 sha256=c65ef7f5f83520cde48bb03d7fce0618d4ac860519f50e25dddbc95d91d6d366\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-rl0mmb5c/wheels/1f/36/7a/0f641175add91ee2acac0e59ab6ed9b3b04069e89b555425a4\n",
            "  Building wheel for ir_measures (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for ir_measures: filename=ir_measures-0.3.3-py3-none-any.whl size=61182 sha256=4e09c91ee6853d2a844e343a284419730b127af9625de39a6ea85778df43ae97\n",
            "  Stored in directory: /root/.cache/pip/wheels/9f/0e/22/718279f23fef1673a4c5e433881c25080a6afaa147e007183e\n",
            "  Building wheel for warc3-wet-clueweb09 (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for warc3-wet-clueweb09: filename=warc3_wet_clueweb09-0.2.5-py3-none-any.whl size=18919 sha256=c86dcadaa8a9bcde8f036a49157c32be84b4c8691085c43f63d4f562e39c4b2a\n",
            "  Stored in directory: /root/.cache/pip/wheels/1a/d7/91/7ffb991df87e62355d945745035470ba2616aa3d83a250b5f9\n",
            "  Building wheel for zlib-state (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for zlib-state: filename=zlib_state-0.1.6-cp310-cp310-linux_x86_64.whl size=21159 sha256=db1c643065cb42a88f2d128808bc491b4f48acfadaddc724acc3eb86d7bc43f3\n",
            "  Stored in directory: /root/.cache/pip/wheels/32/72/7e/aff80f26e926b6e1fb08dfb52aba03c0e058f5e2258deb50a9\n",
            "  Building wheel for chest (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for chest: filename=chest-0.2.3-py3-none-any.whl size=7612 sha256=078ed15b5daec8343ff56a09e4ac46a33abd98ca48bb0ed3e1adc00bb7949d1f\n",
            "  Stored in directory: /root/.cache/pip/wheels/88/cf/99/4773b31f855f9ecedc32a0ae400f7a4a3001b37c439b6d1a73\n",
            "  Building wheel for wget (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for wget: filename=wget-3.2-py3-none-any.whl size=9655 sha256=610b9e482d7c5682469d55d8acadb8c7a3431d3d81073e405c8241564d2c160c\n",
            "  Stored in directory: /root/.cache/pip/wheels/8b/f1/7f/5c94f0a7a505ca1c81cd1d9208ae2064675d97582078e6c769\n",
            "  Building wheel for cbor (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for cbor: filename=cbor-1.0.0-cp310-cp310-linux_x86_64.whl size=53430 sha256=d15895a75e7f99c4f599bd36b63fa6ad04d173661a4c192c15b1a3b4d60eaa1f\n",
            "  Stored in directory: /root/.cache/pip/wheels/85/df/c9/b39e40eccaf76dbd218556639a6dc81562226f4c6a64902c85\n",
            "  Building wheel for cwl-eval (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for cwl-eval: filename=cwl_eval-1.0.12-py3-none-any.whl size=38068 sha256=54649bf7b2bcb69c47554c0e0efcb7c7077e2f39a0858416f52d04a45c469f91\n",
            "  Stored in directory: /root/.cache/pip/wheels/3d/c1/94/94a3e5379b1aa8fb7c7f1ad1956305d5edc98ef745b6067d87\n",
            "Successfully built python-terrier tira ir_measures warc3-wet-clueweb09 zlib-state chest wget cbor cwl-eval\n",
            "Installing collected packages: wget, warc3-wet-clueweb09, warc3-wet, typish, pyjnius, multiset, ijson, heapdict, cbor, zlib-state, unlzw3, trec-car-tools, pytrec_eval_terrier, pyautocorpus, nptyping, matchpy, lz4, dill, deprecated, cwl-eval, chest, ir_measures, inscriptis, docker, tira, ir-datasets, python-terrier\n",
            "Successfully installed cbor-1.0.0 chest-0.2.3 cwl-eval-1.0.12 deprecated-1.2.14 dill-0.3.8 docker-6.1.3 heapdict-1.0.1 ijson-3.2.3 inscriptis-2.4.0.1 ir-datasets-0.5.6 ir_measures-0.3.3 lz4-4.3.3 matchpy-0.5.5 multiset-2.1.1 nptyping-1.4.4 pyautocorpus-0.1.12 pyjnius-1.6.1 python-terrier-0.10.0 pytrec_eval_terrier-0.5.6 tira-0.0.105 trec-car-tools-2.6 typish-1.9.3 unlzw3-0.2.2 warc3-wet-0.2.3 warc3-wet-clueweb09-0.2.5 wget-3.2 zlib-state-0.1.6\n"
          ]
        }
      ],
      "source": [
        "!pip3 install python-terrier ir-datasets git+https://github.com/tira-io/tira.git@development#\\&subdirectory=python-client"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from tira.third_party_integrations import ensure_pyterrier_is_loaded\n",
        "ensure_pyterrier_is_loaded()\n",
        "import pyterrier as pt\n",
        "from tqdm import tqdm\n",
        "import pandas as pd\n",
        "\n",
        "from tira.rest_api_client import Client\n",
        "tira = Client()"
      ],
      "metadata": {
        "id": "vQHGYDqu6HG-",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d32287dd-604e-4771-a1ff-f62c67402cb9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "terrier-assemblies 5.7 jar-with-dependencies not found, downloading to /root/.pyterrier...\n",
            "Done\n",
            "terrier-python-helper 0.0.7 jar not found, downloading to /root/.pyterrier...\n",
            "Done\n",
            "terrier-prf -SNAPSHOT jar not found, downloading to /root/.pyterrier...\n",
            "Done\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "PyTerrier 0.10.0 has loaded Terrier 5.7 (built by craigm on 2022-11-10 18:30) and terrier-helper 0.0.7\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "dataset = pt.get_dataset(\"irds:ir-benchmarks/antique-test-20230107-training\")\n",
        "topics = dataset.get_topics('query')\n",
        "topics[topics['qid'] == '1783010']"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 81
        },
        "id": "ZMzOJg-L6LPB",
        "outputId": "f701fef1-5d8c-44d7-a13f-17eb240af76e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "       qid              query\n",
              "9  1783010  what is blaphsemy"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-2dd4617f-04e2-4bac-b81c-fd104bb3ad17\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>qid</th>\n",
              "      <th>query</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>1783010</td>\n",
              "      <td>what is blaphsemy</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-2dd4617f-04e2-4bac-b81c-fd104bb3ad17')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-2dd4617f-04e2-4bac-b81c-fd104bb3ad17 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-2dd4617f-04e2-4bac-b81c-fd104bb3ad17');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"topics[topics['qid'] == '1783010']\",\n  \"rows\": 1,\n  \"fields\": [\n    {\n      \"column\": \"qid\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          \"1783010\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"query\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          \"what is blaphsemy\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "index = tira.get_run_output('ir-benchmarks/tira-ir-starter/Index (tira-ir-starter-pyterrier)', 'antique-test-20230107-training') + '/index'\n",
        "index = pt.IndexFactory.of(index)\n",
        "\n",
        "bm25 = pt.BatchRetrieve(index, wmodel=\"BM25\")\n",
        "pl2 = pt.BatchRetrieve(index, wmodel=\"PL2\")"
      ],
      "metadata": {
        "id": "pJhDmZtJEQUs"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "dataset = pt.get_dataset(\"irds:ir-benchmarks/msmarco-passage-trec-dl-2019-judged-20230107-training\")\n",
        "index = tira.get_run_output('ir-benchmarks/tira-ir-starter/Index (tira-ir-starter-pyterrier)', 'msmarco-passage-trec-dl-2019-judged-20230107-training') + '/index'\n",
        "index = pt.IndexFactory.of(index)\n",
        "\n",
        "bm25 = pt.BatchRetrieve(index, wmodel=\"BM25\")\n",
        "pl2 = pt.BatchRetrieve(index, wmodel=\"PL2\")"
      ],
      "metadata": {
        "id": "kTSy-b4xdahF",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a236814e-7e90-4f8c-b8eb-ac1077810b6d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Download from the Incubator: https://files.webis.de/data-in-production/data-research/tira-zenodo-dump-preparation/pyterrier-indexes/2023-01-07-22-09-56.zip\n",
            "\tThis is only used for last spot checks before archival to Zenodo.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Download: 100%|██████████| 892M/892M [00:01<00:00, 581MiB/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Download finished. Extract...\n",
            "Extraction finished:  /root/.tira/extracted_runs/ir-benchmarks/msmarco-passage-trec-dl-2019-judged-20230107-training/tira-ir-starter\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tokeniser = pt.autoclass(\"org.terrier.indexing.tokenisation.Tokeniser\").getTokeniser()\n",
        "def pt_tokenise(text):\n",
        "    return ' '.join(tokeniser.getTokens(text))\n"
      ],
      "metadata": {
        "id": "NpJGSKfcEkIW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "hunspell = tira.pt.transform_queries('workshop-on-open-web-search/qspell/hunspell', 'msmarco-passage-trec-dl-2019-judged-20230107-training')\n",
        "pyspell = tira.pt.transform_queries('workshop-on-open-web-search/qspell/pyspell', 'msmarco-passage-trec-dl-2019-judged-20230107-training')\n",
        "rushton = tira.pt.transform_queries('workshop-on-open-web-search/qspell/rushton', 'msmarco-passage-trec-dl-2019-judged-20230107-training')\n",
        "chat_gpt = tira.pt.transform_queries('ir-benchmarks/qspell/ChatGPT', 'msmarco-passage-trec-dl-2019-judged-20230107-training')\n",
        "toks = pt.apply.query(lambda i: pt_tokenise(i['query']))"
      ],
      "metadata": {
        "id": "G7FUu6lSUPnt",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ca28c67a-d47d-495f-af5b-896d90966778"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Download: 3.67kiB [00:00, 25.3MiB/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Download finished. Extract...\n",
            "Extraction finished:  /root/.tira/extracted_runs/workshop-on-open-web-search/msmarco-passage-trec-dl-2019-judged-20230107-training/qspell\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Download: 3.80kiB [00:00, 25.1MiB/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Download finished. Extract...\n",
            "Extraction finished:  /root/.tira/extracted_runs/workshop-on-open-web-search/msmarco-passage-trec-dl-2019-judged-20230107-training/qspell\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Download: 3.72kiB [00:00, 27.5MiB/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Download finished. Extract...\n",
            "Extraction finished:  /root/.tira/extracted_runs/workshop-on-open-web-search/msmarco-passage-trec-dl-2019-judged-20230107-training/qspell\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Download: 3.60kiB [00:00, 25.1MiB/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Download finished. Extract...\n",
            "Extraction finished:  /root/.tira/extracted_runs/ir-benchmarks/msmarco-passage-trec-dl-2019-judged-20230107-training/qspell\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "systems = [\n",
        "    bm25,\n",
        "    pl2,\n",
        "    hunspell >> toks >> bm25,\n",
        "    hunspell >> toks >> pl2,\n",
        "    pyspell >> toks >> bm25,\n",
        "    pyspell >> toks >> pl2,\n",
        "    rushton >> toks >> bm25,\n",
        "    rushton >> toks >> pl2,\n",
        "    chat_gpt >> toks >> bm25,\n",
        "    chat_gpt >> toks >> pl2,\n",
        "]\n",
        "names = ['BM25', 'PL2', 'hunspell >> BM25', 'hunspell >> PL2', 'pyspell >> BM25', 'pyspell >> PL2', 'rushton >> BM25', 'rushton >> PL2', 'ChatGPT >> BM25', 'ChatGPT >> PL2']\n",
        "\n",
        "pt.Experiment(systems, dataset.get_topics('query'), dataset.get_qrels(), ['ndcg_cut.10'], names=names, verbose=True)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 395,
          "referenced_widgets": [
            "a1d1ebaeff0247279f721274a13dd0ae",
            "2fc1f39b71294399aa512e98247ea7a2",
            "eacf9e7b6641464a9e1f3364b20c6cd1",
            "fb33580cf6cc48c2a2f1b19142892baa",
            "21c71ecb34f142cb8440f2af712189a6",
            "49b4ed4ec1eb4d9cbf425c562994638d",
            "5c3b573dfa314012bc278d6c68e054b4",
            "90d17596f92849b7b65eb5eb4ec6f3b3",
            "2e6ba146380b4b318383a717c5f1dad4",
            "73df9e84e1934150b02bb4743e6ae2fb",
            "37f14440d16d4f63a5997f9f0b345367"
          ]
        },
        "id": "t8ivJIj0_-HR",
        "outputId": "e69e1436-fb5e-4e77-a68e-1267f8f20aab"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "pt.Experiment:   0%|          | 0/10 [00:00<?, ?system/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "a1d1ebaeff0247279f721274a13dd0ae"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "               name  ndcg_cut.10\n",
              "0              BM25     0.479540\n",
              "1               PL2     0.466875\n",
              "2  hunspell >> BM25     0.356071\n",
              "3   hunspell >> PL2     0.337754\n",
              "4   pyspell >> BM25     0.312993\n",
              "5    pyspell >> PL2     0.302877\n",
              "6   rushton >> BM25     0.482312\n",
              "7    rushton >> PL2     0.467155\n",
              "8   ChatGPT >> BM25     0.478834\n",
              "9    ChatGPT >> PL2     0.465025"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-ee00b76b-29b7-42e4-818e-d2af107a6ed1\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>name</th>\n",
              "      <th>ndcg_cut.10</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>BM25</td>\n",
              "      <td>0.479540</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>PL2</td>\n",
              "      <td>0.466875</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>hunspell &gt;&gt; BM25</td>\n",
              "      <td>0.356071</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>hunspell &gt;&gt; PL2</td>\n",
              "      <td>0.337754</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>pyspell &gt;&gt; BM25</td>\n",
              "      <td>0.312993</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>pyspell &gt;&gt; PL2</td>\n",
              "      <td>0.302877</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>rushton &gt;&gt; BM25</td>\n",
              "      <td>0.482312</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>rushton &gt;&gt; PL2</td>\n",
              "      <td>0.467155</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>ChatGPT &gt;&gt; BM25</td>\n",
              "      <td>0.478834</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>ChatGPT &gt;&gt; PL2</td>\n",
              "      <td>0.465025</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-ee00b76b-29b7-42e4-818e-d2af107a6ed1')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-ee00b76b-29b7-42e4-818e-d2af107a6ed1 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-ee00b76b-29b7-42e4-818e-d2af107a6ed1');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-7611be20-d430-4f75-abee-da3c3f487af0\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-7611be20-d430-4f75-abee-da3c3f487af0')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-7611be20-d430-4f75-abee-da3c3f487af0 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"pt\",\n  \"rows\": 10,\n  \"fields\": [\n    {\n      \"column\": \"name\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 10,\n        \"samples\": [\n          \"ChatGPT >> BM25\",\n          \"PL2\",\n          \"pyspell >> PL2\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"ndcg_cut.10\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.07681247902817412,\n        \"min\": 0.3028768590504192,\n        \"max\": 0.48231211365691234,\n        \"num_unique_values\": 10,\n        \"samples\": [\n          0.47883421045815394,\n          0.4668753516097153,\n          0.3028768590504192\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "hunspell = tira.pt.transform_queries('workshop-on-open-web-search/qspell/hunspell', 'msmarco-passage-trec-dl-2020-judged-20230107-training')\n",
        "pyspell = tira.pt.transform_queries('workshop-on-open-web-search/qspell/pyspell', 'msmarco-passage-trec-dl-2020-judged-20230107-training')\n",
        "rushton = tira.pt.transform_queries('workshop-on-open-web-search/qspell/rushton', 'msmarco-passage-trec-dl-2020-judged-20230107-training')\n",
        "chat_gpt = tira.pt.transform_queries('ir-benchmarks/qspell/ChatGPT', 'msmarco-passage-trec-dl-2020-judged-20230107-training')\n",
        "toks = pt.apply.query(lambda i: pt_tokenise(i['query']))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YmQRV2fdGiTu",
        "outputId": "d1792b3b-9d09-4d54-d208-519da8d153b7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Download: 3.97kiB [00:00, 3.01MiB/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Download finished. Extract...\n",
            "Extraction finished:  /root/.tira/extracted_runs/workshop-on-open-web-search/msmarco-passage-trec-dl-2020-judged-20230107-training/qspell\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Download: 4.11kiB [00:00, 3.22MiB/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Download finished. Extract...\n",
            "Extraction finished:  /root/.tira/extracted_runs/workshop-on-open-web-search/msmarco-passage-trec-dl-2020-judged-20230107-training/qspell\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Download: 4.00kiB [00:00, 29.6MiB/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Download finished. Extract...\n",
            "Extraction finished:  /root/.tira/extracted_runs/workshop-on-open-web-search/msmarco-passage-trec-dl-2020-judged-20230107-training/qspell\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Download: 3.90kiB [00:00, 8.72MiB/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Download finished. Extract...\n",
            "Extraction finished:  /root/.tira/extracted_runs/ir-benchmarks/msmarco-passage-trec-dl-2020-judged-20230107-training/qspell\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "dataset = pt.get_dataset(\"irds:ir-benchmarks/msmarco-passage-trec-dl-2020-judged-20230107-training\")\n",
        "systems = [\n",
        "    bm25,\n",
        "    pl2,\n",
        "    hunspell >> toks >> bm25,\n",
        "    hunspell >> toks >> pl2,\n",
        "    pyspell >> toks >> bm25,\n",
        "    pyspell >> toks >> pl2,\n",
        "    rushton >> toks >> bm25,\n",
        "    rushton >> toks >> pl2,\n",
        "    chat_gpt >> toks >> bm25,\n",
        "    chat_gpt >> toks >> pl2,\n",
        "]\n",
        "names = ['BM25', 'PL2', 'hunspell >> BM25', 'hunspell >> PL2', 'pyspell >> BM25', 'pyspell >> PL2', 'rushton >> BM25', 'rushton >> PL2', 'ChatGPT >> BM25', 'ChatGPT >> PL2']\n",
        "\n",
        "pt.Experiment(systems, dataset.get_topics('query'), dataset.get_qrels(), ['ndcg_cut.10'], names=names, verbose=True)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 447,
          "referenced_widgets": [
            "8bfe12cbd0a24e0bb317792e8131881e",
            "6c24a096499b4fd7ac5c396552b2114b",
            "901b4a15631e4ec1a92ff401054c2a29",
            "dbc2241bcbc74d3e9bc76cd7c4ca859d",
            "dc59551001e042699fab159425307c21",
            "7226dcc59ee4499f9c703ee6d4a1e76d",
            "6348acf11cd346b3a88af185789cd17a",
            "596540bb73f942eb983764762666ed7f",
            "0a0a4b5cd8444a6290a9629b17c0191a",
            "84d401a4455c4951aed9a8589276cdd9",
            "f920fa10e3404730bf804f1e32957a45"
          ]
        },
        "id": "pLa3pEK2GqNL",
        "outputId": "4acc7e0f-da0d-4ec4-b834-c5edcabb66c0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Download: 235kiB [00:00, 111MiB/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Download finished. Extract...\n",
            "Extraction finished:  /root/.tira/extracted_datasets/ir-benchmarks/msmarco-passage-trec-dl-2020-judged-20230107-training/\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "pt.Experiment:   0%|          | 0/10 [00:00<?, ?system/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "8bfe12cbd0a24e0bb317792e8131881e"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "               name  ndcg_cut.10\n",
              "0              BM25     0.493627\n",
              "1               PL2     0.483847\n",
              "2  hunspell >> BM25     0.417859\n",
              "3   hunspell >> PL2     0.408410\n",
              "4   pyspell >> BM25     0.333832\n",
              "5    pyspell >> PL2     0.324871\n",
              "6   rushton >> BM25     0.493627\n",
              "7    rushton >> PL2     0.483847\n",
              "8   ChatGPT >> BM25     0.482022\n",
              "9    ChatGPT >> PL2     0.470418"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-7faeb9e0-2b01-403a-ad3b-e721cb547b3a\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>name</th>\n",
              "      <th>ndcg_cut.10</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>BM25</td>\n",
              "      <td>0.493627</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>PL2</td>\n",
              "      <td>0.483847</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>hunspell &gt;&gt; BM25</td>\n",
              "      <td>0.417859</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>hunspell &gt;&gt; PL2</td>\n",
              "      <td>0.408410</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>pyspell &gt;&gt; BM25</td>\n",
              "      <td>0.333832</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>pyspell &gt;&gt; PL2</td>\n",
              "      <td>0.324871</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>rushton &gt;&gt; BM25</td>\n",
              "      <td>0.493627</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>rushton &gt;&gt; PL2</td>\n",
              "      <td>0.483847</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>ChatGPT &gt;&gt; BM25</td>\n",
              "      <td>0.482022</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>ChatGPT &gt;&gt; PL2</td>\n",
              "      <td>0.470418</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-7faeb9e0-2b01-403a-ad3b-e721cb547b3a')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-7faeb9e0-2b01-403a-ad3b-e721cb547b3a button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-7faeb9e0-2b01-403a-ad3b-e721cb547b3a');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-cce8eb07-c19c-4b28-82c8-e71916d3236e\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-cce8eb07-c19c-4b28-82c8-e71916d3236e')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-cce8eb07-c19c-4b28-82c8-e71916d3236e button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"pt\",\n  \"rows\": 10,\n  \"fields\": [\n    {\n      \"column\": \"name\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 10,\n        \"samples\": [\n          \"ChatGPT >> BM25\",\n          \"PL2\",\n          \"pyspell >> PL2\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"ndcg_cut.10\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.06523186551744799,\n        \"min\": 0.32487114615026236,\n        \"max\": 0.4936273783432555,\n        \"num_unique_values\": 8,\n        \"samples\": [\n          0.4838474001303005,\n          0.32487114615026236,\n          0.4936273783432555\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "systems = [\n",
        "    bm25,\n",
        "    pl2,\n",
        "    hunspell >> toks >> bm25,\n",
        "    hunspell >> toks >> pl2,\n",
        "    pyspell >> toks >> bm25,\n",
        "    pyspell >> toks >> pl2,\n",
        "    rushton >> toks >> bm25,\n",
        "    rushton >> toks >> pl2,\n",
        "    chat_gpt >> toks >> bm25,\n",
        "    chat_gpt >> toks >> pl2,\n",
        "]\n",
        "names = ['BM25', 'PL2', 'hunspell >> BM25', 'hunspell >> PL2', 'pyspell >> BM25', 'pyspell >> PL2', 'rushton >> BM25', 'rushton >> PL2', 'ChatGPT >> BM25', 'ChatGPT >> PL2']\n",
        "\n",
        "pt.Experiment(systems, dataset.get_topics('query'), dataset.get_qrels(), ['ndcg_cut.10'], names=names, verbose=True)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 395,
          "referenced_widgets": [
            "4e7d0630b400492996bbfc64fdb64472",
            "7d1d5591ed3d4e289b258f1ee0a87591",
            "5ec88ae212e2434e9b8df37fd06d27e6",
            "b069d5056aed4b12b5781ecde3b09ad3",
            "73926514d6cc4fd4889c776e3df168ab",
            "2b35fc599bfc47948bef702a0ad172b0",
            "05a98a12bbbe4b2e98e81afa6cbc30d0",
            "431b8ccf27aa4226b8822ee415f6b8cb",
            "dfba6879893d4b16a1792afb819b3ff2",
            "6c18f9f2716248e6a614318ca8399d56",
            "42fc256c1eae47fe9977b1743f962afb"
          ]
        },
        "id": "0ETdLYQ8EpKT",
        "outputId": "43699a3d-7afc-409c-c0cd-2c7419c346be"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "pt.Experiment:   0%|          | 0/10 [00:00<?, ?system/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "4e7d0630b400492996bbfc64fdb64472"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "               name  ndcg_cut.10\n",
              "0              BM25     0.510402\n",
              "1               PL2     0.488955\n",
              "2  hunspell >> BM25     0.480913\n",
              "3   hunspell >> PL2     0.458890\n",
              "4   pyspell >> BM25     0.433223\n",
              "5    pyspell >> PL2     0.410252\n",
              "6   rushton >> BM25     0.509530\n",
              "7    rushton >> PL2     0.488457\n",
              "8   ChatGPT >> BM25     0.490155\n",
              "9    ChatGPT >> PL2     0.470942"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-82a46adf-7281-44d6-b3ee-033b79c64328\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>name</th>\n",
              "      <th>ndcg_cut.10</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>BM25</td>\n",
              "      <td>0.510402</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>PL2</td>\n",
              "      <td>0.488955</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>hunspell &gt;&gt; BM25</td>\n",
              "      <td>0.480913</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>hunspell &gt;&gt; PL2</td>\n",
              "      <td>0.458890</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>pyspell &gt;&gt; BM25</td>\n",
              "      <td>0.433223</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>pyspell &gt;&gt; PL2</td>\n",
              "      <td>0.410252</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>rushton &gt;&gt; BM25</td>\n",
              "      <td>0.509530</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>rushton &gt;&gt; PL2</td>\n",
              "      <td>0.488457</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>ChatGPT &gt;&gt; BM25</td>\n",
              "      <td>0.490155</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>ChatGPT &gt;&gt; PL2</td>\n",
              "      <td>0.470942</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-82a46adf-7281-44d6-b3ee-033b79c64328')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-82a46adf-7281-44d6-b3ee-033b79c64328 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-82a46adf-7281-44d6-b3ee-033b79c64328');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-9e1a0810-154f-48fc-9819-842f4cd68b69\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-9e1a0810-154f-48fc-9819-842f4cd68b69')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-9e1a0810-154f-48fc-9819-842f4cd68b69 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"pt\",\n  \"rows\": 10,\n  \"fields\": [\n    {\n      \"column\": \"name\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 10,\n        \"samples\": [\n          \"ChatGPT >> BM25\",\n          \"PL2\",\n          \"pyspell >> PL2\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"ndcg_cut.10\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.03213297119389331,\n        \"min\": 0.4102523952775302,\n        \"max\": 0.510402147058545,\n        \"num_unique_values\": 10,\n        \"samples\": [\n          0.49015465567977884,\n          0.4889550318738034,\n          0.4102523952775302\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Oracle"
      ],
      "metadata": {
        "id": "gT02vNFZ1JH9"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "dataset = pt.get_dataset(\"irds:ir-benchmarks/msmarco-passage-trec-dl-2020-judged-20230107-training\")\n",
        "systems = [\n",
        "    pl2,\n",
        "    hunspell >> toks >> pl2,\n",
        "    pyspell >> toks >> pl2,\n",
        "    rushton >> toks >> pl2,\n",
        "    chat_gpt >> toks >> pl2,\n",
        "]\n",
        "names = ['PL2', 'hunspell >> PL2', 'pyspell >> PL2', 'rushton >> PL2', 'ChatGPT >> PL2']\n",
        "df_eval = pt.Experiment(systems, dataset.get_topics('query'), dataset.get_qrels(), ['ndcg_cut.10'], names=names, verbose=True, perquery=True)\n",
        "df_eval"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 509,
          "referenced_widgets": [
            "fae8aaab07f04bce9a601549a3124bc6",
            "ec4b89232da94970a1f7288b1e29afa2",
            "486d035e687441578d23e7eb2e47a5aa",
            "c997c5b8cc1842c886b8dda6a6710ef9",
            "38dfa4b9d8fb43589e8bcdf75f0dd5b5",
            "290f7c66a3fa418eb500b3232226461e",
            "dc2f2f0ac9874f48be7038e6bc74dcc1",
            "575739d54daa4fcd96029274ef29a788",
            "686430e8f0714318ac4a784f23e17b24",
            "90cf03d22a77451383e6ddf7d1069d14",
            "74a086355ce44703b61c816d702bd681"
          ]
        },
        "id": "sjfKT-itjotL",
        "outputId": "d69350cf-d300-41a7-b989-9b927d0a4775"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Download: 235kiB [00:00, 168MiB/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Download finished. Extract...\n",
            "Extraction finished:  /root/.tira/extracted_datasets/ir-benchmarks/msmarco-passage-trec-dl-2020-judged-20230107-training/\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "pt.Experiment:   0%|          | 0/5 [00:00<?, ?system/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "fae8aaab07f04bce9a601549a3124bc6"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "               name      qid      measure     value\n",
              "216  ChatGPT >> PL2  1030303  ndcg_cut.10  0.872746\n",
              "217  ChatGPT >> PL2  1037496  ndcg_cut.10  0.520479\n",
              "218  ChatGPT >> PL2  1043135  ndcg_cut.10  0.601138\n",
              "219  ChatGPT >> PL2  1051399  ndcg_cut.10  0.164867\n",
              "220  ChatGPT >> PL2  1064670  ndcg_cut.10  0.773036\n",
              "..              ...      ...          ...       ...\n",
              "211  rushton >> PL2   911232  ndcg_cut.10  0.670264\n",
              "212  rushton >> PL2   914916  ndcg_cut.10  0.512427\n",
              "213  rushton >> PL2   938400  ndcg_cut.10  0.128349\n",
              "214  rushton >> PL2   940547  ndcg_cut.10  0.165437\n",
              "215  rushton >> PL2   997622  ndcg_cut.10  0.352664\n",
              "\n",
              "[270 rows x 4 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-b3c235c1-2903-438b-9e44-1600b9700805\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>name</th>\n",
              "      <th>qid</th>\n",
              "      <th>measure</th>\n",
              "      <th>value</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>216</th>\n",
              "      <td>ChatGPT &gt;&gt; PL2</td>\n",
              "      <td>1030303</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.872746</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>217</th>\n",
              "      <td>ChatGPT &gt;&gt; PL2</td>\n",
              "      <td>1037496</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.520479</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>218</th>\n",
              "      <td>ChatGPT &gt;&gt; PL2</td>\n",
              "      <td>1043135</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.601138</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>219</th>\n",
              "      <td>ChatGPT &gt;&gt; PL2</td>\n",
              "      <td>1051399</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.164867</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>220</th>\n",
              "      <td>ChatGPT &gt;&gt; PL2</td>\n",
              "      <td>1064670</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.773036</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>211</th>\n",
              "      <td>rushton &gt;&gt; PL2</td>\n",
              "      <td>911232</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.670264</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>212</th>\n",
              "      <td>rushton &gt;&gt; PL2</td>\n",
              "      <td>914916</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.512427</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>213</th>\n",
              "      <td>rushton &gt;&gt; PL2</td>\n",
              "      <td>938400</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.128349</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>214</th>\n",
              "      <td>rushton &gt;&gt; PL2</td>\n",
              "      <td>940547</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.165437</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>215</th>\n",
              "      <td>rushton &gt;&gt; PL2</td>\n",
              "      <td>997622</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.352664</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>270 rows × 4 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-b3c235c1-2903-438b-9e44-1600b9700805')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-b3c235c1-2903-438b-9e44-1600b9700805 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-b3c235c1-2903-438b-9e44-1600b9700805');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-37f6821e-9a37-4273-a024-3c5a5d0341d7\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-37f6821e-9a37-4273-a024-3c5a5d0341d7')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-37f6821e-9a37-4273-a024-3c5a5d0341d7 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "df_eval",
              "summary": "{\n  \"name\": \"df_eval\",\n  \"rows\": 270,\n  \"fields\": [\n    {\n      \"column\": \"name\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 5,\n        \"samples\": [\n          \"PL2\",\n          \"rushton >> PL2\",\n          \"hunspell >> PL2\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"qid\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 54,\n        \"samples\": [\n          \"1133579\",\n          \"911232\",\n          \"877809\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"measure\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          \"ndcg_cut.10\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"value\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.2896920349030595,\n        \"min\": 0.0,\n        \"max\": 1.0,\n        \"num_unique_values\": 67,\n        \"samples\": [\n          0.04034931212248002\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# The oracle (that always chooses the best variant out of \"no spelling correction\", \"hunspell\", \"rusthon\", \"chatGPT\")\n",
        "df_eval.groupby('qid').agg({'value': 'max'}).reset_index()['value'].mean()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8VRF_WZbj4iU",
        "outputId": "1b6d2e01-284d-42a3-9852-72fa74767c18"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.49665603388228124"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "dataset = pt.get_dataset(\"irds:ir-benchmarks/msmarco-passage-trec-dl-2019-judged-20230107-training\")\n",
        "systems = [\n",
        "    pl2,\n",
        "    hunspell >> toks >> pl2,\n",
        "    pyspell >> toks >> pl2,\n",
        "    rushton >> toks >> pl2,\n",
        "    chat_gpt >> toks >> pl2,\n",
        "]\n",
        "names = ['PL2', 'hunspell >> PL2', 'pyspell >> PL2', 'rushton >> PL2', 'ChatGPT >> PL2']\n",
        "df_eval = pt.Experiment(systems, dataset.get_topics('query'), dataset.get_qrels(), ['ndcg_cut.10'], names=names, verbose=True, perquery=True)\n",
        "df_eval"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 509,
          "referenced_widgets": [
            "b2b5a240658349c891bcddf419df6971",
            "4a95a1669d6b42b8a584ff576a06073e",
            "f442063c7e54430bba7ec02c0c2ee54c",
            "8441b160fd5a4b11a49ca4d7e3bbc3df",
            "31b88a7fcb0d4fd7bdaea441b8737a19",
            "7c7859222aa3449e83aeaacb3ecdce19",
            "1514f1eb6e4c4f9d8904dc5b1b4c8f07",
            "02f99b0379744100999869fc36cf9e30",
            "7509d5f93bf64d85946d4ec2516e383e",
            "d2916746d061427ab6f560f03a8bac47",
            "00bb6162e83c44599dbc50d5a1e4a1fc"
          ]
        },
        "id": "dU_4eNx8giNf",
        "outputId": "f8842f91-409c-4aee-eed7-08c8123df6a9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Download: 199kiB [00:00, 162MiB/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Download finished. Extract...\n",
            "Extraction finished:  /root/.tira/extracted_datasets/ir-benchmarks/msmarco-passage-trec-dl-2019-judged-20230107-training/\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "pt.Experiment:   0%|          | 0/5 [00:00<?, ?system/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "b2b5a240658349c891bcddf419df6971"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "               name      qid      measure     value\n",
              "181  ChatGPT >> PL2  1037798  ndcg_cut.10  0.131672\n",
              "193  ChatGPT >> PL2   104861  ndcg_cut.10  0.745783\n",
              "174  ChatGPT >> PL2  1063750  ndcg_cut.10  0.112485\n",
              "201  ChatGPT >> PL2  1103812  ndcg_cut.10  0.501800\n",
              "198  ChatGPT >> PL2  1106007  ndcg_cut.10  0.375959\n",
              "..              ...      ...          ...       ...\n",
              "163  rushton >> PL2   855410  ndcg_cut.10  0.881199\n",
              "159  rushton >> PL2    87181  ndcg_cut.10  0.583667\n",
              "162  rushton >> PL2    87452  ndcg_cut.10  0.456956\n",
              "139  rushton >> PL2   915593  ndcg_cut.10  0.475575\n",
              "142  rushton >> PL2   962179  ndcg_cut.10  0.000000\n",
              "\n",
              "[215 rows x 4 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-d1c3b7d2-ebc4-4cd7-9637-c9cbb80e6374\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>name</th>\n",
              "      <th>qid</th>\n",
              "      <th>measure</th>\n",
              "      <th>value</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>181</th>\n",
              "      <td>ChatGPT &gt;&gt; PL2</td>\n",
              "      <td>1037798</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.131672</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>193</th>\n",
              "      <td>ChatGPT &gt;&gt; PL2</td>\n",
              "      <td>104861</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.745783</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>174</th>\n",
              "      <td>ChatGPT &gt;&gt; PL2</td>\n",
              "      <td>1063750</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.112485</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>201</th>\n",
              "      <td>ChatGPT &gt;&gt; PL2</td>\n",
              "      <td>1103812</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.501800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>198</th>\n",
              "      <td>ChatGPT &gt;&gt; PL2</td>\n",
              "      <td>1106007</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.375959</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>163</th>\n",
              "      <td>rushton &gt;&gt; PL2</td>\n",
              "      <td>855410</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.881199</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>159</th>\n",
              "      <td>rushton &gt;&gt; PL2</td>\n",
              "      <td>87181</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.583667</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>162</th>\n",
              "      <td>rushton &gt;&gt; PL2</td>\n",
              "      <td>87452</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.456956</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>139</th>\n",
              "      <td>rushton &gt;&gt; PL2</td>\n",
              "      <td>915593</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.475575</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>142</th>\n",
              "      <td>rushton &gt;&gt; PL2</td>\n",
              "      <td>962179</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>215 rows × 4 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-d1c3b7d2-ebc4-4cd7-9637-c9cbb80e6374')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-d1c3b7d2-ebc4-4cd7-9637-c9cbb80e6374 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-d1c3b7d2-ebc4-4cd7-9637-c9cbb80e6374');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-c4e4fe45-417f-4ca9-bf41-5208bda74222\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-c4e4fe45-417f-4ca9-bf41-5208bda74222')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-c4e4fe45-417f-4ca9-bf41-5208bda74222 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "df_eval",
              "summary": "{\n  \"name\": \"df_eval\",\n  \"rows\": 215,\n  \"fields\": [\n    {\n      \"column\": \"name\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 5,\n        \"samples\": [\n          \"PL2\",\n          \"rushton >> PL2\",\n          \"hunspell >> PL2\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"qid\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 43,\n        \"samples\": [\n          \"833860\",\n          \"183378\",\n          \"19335\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"measure\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          \"ndcg_cut.10\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"value\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.284214292178092,\n        \"min\": 0.0,\n        \"max\": 1.0,\n        \"num_unique_values\": 50,\n        \"samples\": [\n          0.0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# The oracle (that always chooses the best variant out of \"no spelling correction\", \"hunspell\", \"rusthon\", \"chatGPT\")\n",
        "df_eval.groupby('qid').agg({'value': 'max'}).reset_index()['value'].mean()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pIxRUhZ8ilUi",
        "outputId": "fee217a7-50f2-44f0-937b-d4b18570b531"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.4671554027880264"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "systems = [\n",
        "    bm25,\n",
        "    hunspell >> toks >> bm25,\n",
        "    pyspell >> toks >> bm25,\n",
        "    rushton >> toks >> bm25,\n",
        "    chat_gpt >> toks >> bm25,\n",
        "]\n",
        "names = ['BM25', 'hunspell >> BM25', 'pyspell >> BM25', 'rushton >> BM25', 'ChatGPT >> BM25']\n",
        "df_eval = pt.Experiment(systems, dataset.get_topics('query'), dataset.get_qrels(), ['ndcg_cut.10'], names=names, verbose=True, perquery=True)\n",
        "df_eval"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 456,
          "referenced_widgets": [
            "dbde6585132a44c9a0fd36e9983d5437",
            "0901902d78ca4bcdb565973e9481f43b",
            "09deff9aff8149358dacb8ee9988b587",
            "15908a55914644e7a68e840cd5b36583",
            "67f986cf6bb24ceaaa39ddd3dbf6c6c0",
            "a489ea48509e421183a31c80587dac25",
            "7e825ae484664cc9807bd1bce32190d6",
            "ed98202dafc640b98ddc00ed18ec2871",
            "4549dbd08e4c45f4b6811d56679f5e90",
            "0253cbce570a44f4b60bee1b71a4b98a",
            "9f4b0fde314f4b9096fdd678e4c8bb43"
          ]
        },
        "id": "-a1DuxOfHGcF",
        "outputId": "62e0b794-c220-4596-ef1b-8155abb86ac5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "pt.Experiment:   0%|          | 0/5 [00:00<?, ?system/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "dbde6585132a44c9a0fd36e9983d5437"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                name      qid      measure     value\n",
              "0               BM25  1030303  ndcg_cut.10  0.872746\n",
              "1               BM25  1037496  ndcg_cut.10  0.542564\n",
              "2               BM25  1043135  ndcg_cut.10  0.595064\n",
              "3               BM25  1051399  ndcg_cut.10  0.179623\n",
              "4               BM25  1064670  ndcg_cut.10  0.771352\n",
              "..               ...      ...          ...       ...\n",
              "211  rushton >> BM25   911232  ndcg_cut.10  0.670264\n",
              "212  rushton >> BM25   914916  ndcg_cut.10  0.512427\n",
              "213  rushton >> BM25   938400  ndcg_cut.10  0.114339\n",
              "214  rushton >> BM25   940547  ndcg_cut.10  0.127349\n",
              "215  rushton >> BM25   997622  ndcg_cut.10  0.481900\n",
              "\n",
              "[270 rows x 4 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-c2c64079-6f62-4801-a6ef-73525e0d08f3\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>name</th>\n",
              "      <th>qid</th>\n",
              "      <th>measure</th>\n",
              "      <th>value</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>BM25</td>\n",
              "      <td>1030303</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.872746</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>BM25</td>\n",
              "      <td>1037496</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.542564</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>BM25</td>\n",
              "      <td>1043135</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.595064</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>BM25</td>\n",
              "      <td>1051399</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.179623</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>BM25</td>\n",
              "      <td>1064670</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.771352</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>211</th>\n",
              "      <td>rushton &gt;&gt; BM25</td>\n",
              "      <td>911232</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.670264</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>212</th>\n",
              "      <td>rushton &gt;&gt; BM25</td>\n",
              "      <td>914916</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.512427</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>213</th>\n",
              "      <td>rushton &gt;&gt; BM25</td>\n",
              "      <td>938400</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.114339</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>214</th>\n",
              "      <td>rushton &gt;&gt; BM25</td>\n",
              "      <td>940547</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.127349</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>215</th>\n",
              "      <td>rushton &gt;&gt; BM25</td>\n",
              "      <td>997622</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.481900</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>270 rows × 4 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-c2c64079-6f62-4801-a6ef-73525e0d08f3')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-c2c64079-6f62-4801-a6ef-73525e0d08f3 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-c2c64079-6f62-4801-a6ef-73525e0d08f3');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-d7589b44-c90e-46ee-917e-6802fdf1561b\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-d7589b44-c90e-46ee-917e-6802fdf1561b')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-d7589b44-c90e-46ee-917e-6802fdf1561b button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "df_eval",
              "summary": "{\n  \"name\": \"df_eval\",\n  \"rows\": 270,\n  \"fields\": [\n    {\n      \"column\": \"name\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 5,\n        \"samples\": [\n          \"ChatGPT >> BM25\",\n          \"rushton >> BM25\",\n          \"hunspell >> BM25\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"qid\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 54,\n        \"samples\": [\n          \"1133579\",\n          \"911232\",\n          \"877809\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"measure\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          \"ndcg_cut.10\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"value\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.2899593977954373,\n        \"min\": 0.0,\n        \"max\": 1.0,\n        \"num_unique_values\": 68,\n        \"samples\": [\n          0.6555420687894586\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# The oracle (that always chooses the best variant out of \"no spelling correction\", \"hunspell\", \"rusthon\", \"chatGPT\")\n",
        "df_eval.groupby('qid').agg({'value': 'max'}).reset_index()['value'].mean()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hyPxJ7TAHOw1",
        "outputId": "4b4af14d-83a9-4077-959f-4a5ecaa51695"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.5068656659357653"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "systems = [\n",
        "    bm25,\n",
        "    hunspell >> toks >> bm25,\n",
        "    pyspell >> toks >> bm25,\n",
        "    rushton >> toks >> bm25,\n",
        "    chat_gpt >> toks >> bm25,\n",
        "]\n",
        "names = ['BM25', 'hunspell >> BM25', 'pyspell >> BM25', 'rushton >> BM25', 'ChatGPT >> BM25']\n",
        "df_eval = pt.Experiment(systems, dataset.get_topics('query'), dataset.get_qrels(), ['ndcg_cut.10'], names=names, verbose=True, perquery=True)\n",
        "df_eval"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 456,
          "referenced_widgets": [
            "cd92d09025e249f698e35bc7bb573ba1",
            "97c55f41783d4f0aae80a1b25bf19720",
            "0c5d7a44599d4336801961063d3b42d8",
            "0600fa99f9a6486dac4b44ca717455ff",
            "f117c5c455c248d78f63034fc1162254",
            "f3328d097a0b47e981a45cb0dd19d70b",
            "6542c8713aa14a9e8883e4cfb59108e5",
            "83b8e6d8a4494c328bd7138816dd2c29",
            "4c22749ff701428ebf2ca824871010c5",
            "e300abbf67aa408fb8314a97b487655a",
            "544dfb9e84854160a9d45f83ebbf598f"
          ]
        },
        "id": "Pw-rk-FiBmtF",
        "outputId": "0d661453-2743-4058-b536-0b0d89b4b03c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "pt.Experiment:   0%|          | 0/5 [00:00<?, ?system/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "cd92d09025e249f698e35bc7bb573ba1"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                name      qid      measure     value\n",
              "9               BM25  1037798  ndcg_cut.10  0.152866\n",
              "21              BM25   104861  ndcg_cut.10  0.776306\n",
              "2               BM25  1063750  ndcg_cut.10  0.142811\n",
              "29              BM25  1103812  ndcg_cut.10  0.507699\n",
              "26              BM25  1106007  ndcg_cut.10  0.368773\n",
              "..               ...      ...          ...       ...\n",
              "163  rushton >> BM25   855410  ndcg_cut.10  0.881199\n",
              "159  rushton >> BM25    87181  ndcg_cut.10  0.579026\n",
              "162  rushton >> BM25    87452  ndcg_cut.10  0.498614\n",
              "139  rushton >> BM25   915593  ndcg_cut.10  0.417021\n",
              "142  rushton >> BM25   962179  ndcg_cut.10  0.000000\n",
              "\n",
              "[215 rows x 4 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-744d29a1-4af8-473c-9bee-614a7363e418\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>name</th>\n",
              "      <th>qid</th>\n",
              "      <th>measure</th>\n",
              "      <th>value</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>BM25</td>\n",
              "      <td>1037798</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.152866</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21</th>\n",
              "      <td>BM25</td>\n",
              "      <td>104861</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.776306</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>BM25</td>\n",
              "      <td>1063750</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.142811</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>29</th>\n",
              "      <td>BM25</td>\n",
              "      <td>1103812</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.507699</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26</th>\n",
              "      <td>BM25</td>\n",
              "      <td>1106007</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.368773</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>163</th>\n",
              "      <td>rushton &gt;&gt; BM25</td>\n",
              "      <td>855410</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.881199</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>159</th>\n",
              "      <td>rushton &gt;&gt; BM25</td>\n",
              "      <td>87181</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.579026</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>162</th>\n",
              "      <td>rushton &gt;&gt; BM25</td>\n",
              "      <td>87452</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.498614</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>139</th>\n",
              "      <td>rushton &gt;&gt; BM25</td>\n",
              "      <td>915593</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.417021</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>142</th>\n",
              "      <td>rushton &gt;&gt; BM25</td>\n",
              "      <td>962179</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>215 rows × 4 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-744d29a1-4af8-473c-9bee-614a7363e418')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-744d29a1-4af8-473c-9bee-614a7363e418 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-744d29a1-4af8-473c-9bee-614a7363e418');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-72a4d30d-47e2-4c94-8054-bb1eb24799ba\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-72a4d30d-47e2-4c94-8054-bb1eb24799ba')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-72a4d30d-47e2-4c94-8054-bb1eb24799ba button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "df_eval",
              "summary": "{\n  \"name\": \"df_eval\",\n  \"rows\": 215,\n  \"fields\": [\n    {\n      \"column\": \"name\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 5,\n        \"samples\": [\n          \"ChatGPT >> BM25\",\n          \"rushton >> BM25\",\n          \"hunspell >> BM25\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"qid\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 43,\n        \"samples\": [\n          \"833860\",\n          \"183378\",\n          \"19335\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"measure\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          \"ndcg_cut.10\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"value\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.27818718424202615,\n        \"min\": 0.0,\n        \"max\": 1.0,\n        \"num_unique_values\": 52,\n        \"samples\": [\n          0.7490351431774142\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# The oracle (that always chooses the best variant out of \"no spelling correction\", \"hunspell\", \"rusthon\", \"chatGPT\")\n",
        "df_eval.groupby('qid').agg({'value': 'max'}).reset_index()['value'].mean()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UJHd_fMsByBA",
        "outputId": "66cf0a44-fbd4-49b7-88ea-97a650353cee"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.4828546051406672"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "systems = [\n",
        "    bm25,\n",
        "    hunspell >> toks >> bm25,\n",
        "    pyspell >> toks >> bm25,\n",
        "    rushton >> toks >> bm25,\n",
        "    chat_gpt >> toks >> bm25,\n",
        "]\n",
        "names = ['BM25', 'hunspell >> BM25', 'pyspell >> BM25', 'rushton >> BM25', 'ChatGPT >> BM25']\n",
        "df_eval = pt.Experiment(systems, dataset.get_topics('query'), dataset.get_qrels(), ['ndcg_cut.10'], names=names, verbose=True, perquery=True)\n",
        "df_eval"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 456,
          "referenced_widgets": [
            "b86ef2f2bb804898aef52b026f4fe8f3",
            "b445e1dbd2b74592bd4e93bfc976eade",
            "63d409426256491a945aa6e9fadc91e0",
            "093d08d73e614dc28c45610edb97428b",
            "62e8c529708f4579878331d3aa5fdb5c",
            "83197e1f780c4257a39531f194825d46",
            "237e08f82d724184b1dcb9438044c7f8",
            "e037a00aeea44f53b39f8c5af8c38dc7",
            "fcafdfb4ddda44ad9f2f76ce0ad31d8e",
            "0b355dc157164f8fa5dff8c36620e7f1",
            "cb46c352c0df46a594345963d7dc9e77"
          ]
        },
        "id": "eO-flVR51K7y",
        "outputId": "4c445839-011d-472d-a47c-88710bebe12c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "pt.Experiment:   0%|          | 0/5 [00:00<?, ?system/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "b86ef2f2bb804898aef52b026f4fe8f3"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                name      qid      measure     value\n",
              "70              BM25   100653  ndcg_cut.10  0.396720\n",
              "85              BM25  1015624  ndcg_cut.10  0.575231\n",
              "43              BM25  1017690  ndcg_cut.10  0.754357\n",
              "116             BM25  1035857  ndcg_cut.10  0.654812\n",
              "122             BM25   103830  ndcg_cut.10  0.513426\n",
              "..               ...      ...          ...       ...\n",
              "619  rushton >> BM25   851124  ndcg_cut.10  0.577386\n",
              "695  rushton >> BM25   896725  ndcg_cut.10  0.380379\n",
              "704  rushton >> BM25   922849  ndcg_cut.10  0.565539\n",
              "778  rushton >> BM25   949154  ndcg_cut.10  0.664721\n",
              "742  rushton >> BM25   953489  ndcg_cut.10  0.195793\n",
              "\n",
              "[1000 rows x 4 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-d0dbc1df-dff0-4052-87a3-2d66af381546\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>name</th>\n",
              "      <th>qid</th>\n",
              "      <th>measure</th>\n",
              "      <th>value</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>70</th>\n",
              "      <td>BM25</td>\n",
              "      <td>100653</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.396720</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>85</th>\n",
              "      <td>BM25</td>\n",
              "      <td>1015624</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.575231</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>43</th>\n",
              "      <td>BM25</td>\n",
              "      <td>1017690</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.754357</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>116</th>\n",
              "      <td>BM25</td>\n",
              "      <td>1035857</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.654812</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>122</th>\n",
              "      <td>BM25</td>\n",
              "      <td>103830</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.513426</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>619</th>\n",
              "      <td>rushton &gt;&gt; BM25</td>\n",
              "      <td>851124</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.577386</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>695</th>\n",
              "      <td>rushton &gt;&gt; BM25</td>\n",
              "      <td>896725</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.380379</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>704</th>\n",
              "      <td>rushton &gt;&gt; BM25</td>\n",
              "      <td>922849</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.565539</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>778</th>\n",
              "      <td>rushton &gt;&gt; BM25</td>\n",
              "      <td>949154</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.664721</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>742</th>\n",
              "      <td>rushton &gt;&gt; BM25</td>\n",
              "      <td>953489</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.195793</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>1000 rows × 4 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-d0dbc1df-dff0-4052-87a3-2d66af381546')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-d0dbc1df-dff0-4052-87a3-2d66af381546 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-d0dbc1df-dff0-4052-87a3-2d66af381546');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-7036b025-56ee-4dcc-a48d-be50ea7d6ee7\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-7036b025-56ee-4dcc-a48d-be50ea7d6ee7')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-7036b025-56ee-4dcc-a48d-be50ea7d6ee7 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "df_eval",
              "summary": "{\n  \"name\": \"df_eval\",\n  \"rows\": 1000,\n  \"fields\": [\n    {\n      \"column\": \"name\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 5,\n        \"samples\": [\n          \"ChatGPT >> BM25\",\n          \"rushton >> BM25\",\n          \"hunspell >> BM25\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"qid\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 200,\n        \"samples\": [\n          \"2864267\",\n          \"1287437\",\n          \"1623623\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"measure\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          \"ndcg_cut.10\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"value\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.20592548374893424,\n        \"min\": 0.0,\n        \"max\": 0.9765911666162366,\n        \"num_unique_values\": 306,\n        \"samples\": [\n          0.745302598560629\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# The oracle (that always chooses the best variant out of \"no spelling correction\", \"hunspell\", \"rusthon\", \"chatGPT\")\n",
        "df_eval.groupby('qid').agg({'value': 'max'}).reset_index()['value'].mean()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8eS_wInh1tJK",
        "outputId": "f47e429c-72f4-4f1b-803e-940faad155e5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.524358154006209"
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "systems = [\n",
        "    pl2,\n",
        "    hunspell >> toks >> pl2,\n",
        "    pyspell >> toks >> pl2,\n",
        "    rushton >> toks >> pl2,\n",
        "    chat_gpt >> toks >> pl2,\n",
        "]\n",
        "names = ['PL2', 'hunspell >> PL2', 'pyspell >> PL2', 'rushton >> PL2', 'ChatGPT >> PL2']\n",
        "df_eval = pt.Experiment(systems, dataset.get_topics('query'), dataset.get_qrels(), ['ndcg_cut.10'], names=names, verbose=True, perquery=True)\n",
        "df_eval"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 456,
          "referenced_widgets": [
            "fddd7a8e2e214cafb02c1e49634074fe",
            "c2c8c098371f41e7a56a054a809e482d",
            "9a81772c00c24cf2aff7d6bc6255a514",
            "8590ae86846f4123969a12f752db99ef",
            "1292f41a5606410399670c3a24882e2b",
            "64733394b3314329b03b5220f90cf0ba",
            "e0d55e4c78dc4e238c978530d4f43a50",
            "c9f95b527e1b4eeba4cd06a04880685b",
            "1305ae826bd44e28a0c3b1d92aba931b",
            "fbf6d5435b8f46f7831b10e793d5c658",
            "99984cc2bbbd40aeaef689e6b005df10"
          ]
        },
        "id": "DYh20Sqe4F0M",
        "outputId": "44fb9e21-5f12-48e4-c6d1-1e5ea6f7b82f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "pt.Experiment:   0%|          | 0/5 [00:00<?, ?system/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "fddd7a8e2e214cafb02c1e49634074fe"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "               name      qid      measure     value\n",
              "870  ChatGPT >> PL2   100653  ndcg_cut.10  0.396720\n",
              "885  ChatGPT >> PL2  1015624  ndcg_cut.10  0.638852\n",
              "843  ChatGPT >> PL2  1017690  ndcg_cut.10  0.750770\n",
              "916  ChatGPT >> PL2  1035857  ndcg_cut.10  0.645368\n",
              "922  ChatGPT >> PL2   103830  ndcg_cut.10  0.563644\n",
              "..              ...      ...          ...       ...\n",
              "619  rushton >> PL2   851124  ndcg_cut.10  0.537939\n",
              "695  rushton >> PL2   896725  ndcg_cut.10  0.362022\n",
              "704  rushton >> PL2   922849  ndcg_cut.10  0.505162\n",
              "778  rushton >> PL2   949154  ndcg_cut.10  0.664721\n",
              "742  rushton >> PL2   953489  ndcg_cut.10  0.164093\n",
              "\n",
              "[1000 rows x 4 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-79ada582-d014-4d10-837d-ca03101d0e19\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>name</th>\n",
              "      <th>qid</th>\n",
              "      <th>measure</th>\n",
              "      <th>value</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>870</th>\n",
              "      <td>ChatGPT &gt;&gt; PL2</td>\n",
              "      <td>100653</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.396720</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>885</th>\n",
              "      <td>ChatGPT &gt;&gt; PL2</td>\n",
              "      <td>1015624</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.638852</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>843</th>\n",
              "      <td>ChatGPT &gt;&gt; PL2</td>\n",
              "      <td>1017690</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.750770</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>916</th>\n",
              "      <td>ChatGPT &gt;&gt; PL2</td>\n",
              "      <td>1035857</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.645368</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>922</th>\n",
              "      <td>ChatGPT &gt;&gt; PL2</td>\n",
              "      <td>103830</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.563644</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>619</th>\n",
              "      <td>rushton &gt;&gt; PL2</td>\n",
              "      <td>851124</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.537939</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>695</th>\n",
              "      <td>rushton &gt;&gt; PL2</td>\n",
              "      <td>896725</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.362022</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>704</th>\n",
              "      <td>rushton &gt;&gt; PL2</td>\n",
              "      <td>922849</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.505162</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>778</th>\n",
              "      <td>rushton &gt;&gt; PL2</td>\n",
              "      <td>949154</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.664721</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>742</th>\n",
              "      <td>rushton &gt;&gt; PL2</td>\n",
              "      <td>953489</td>\n",
              "      <td>ndcg_cut.10</td>\n",
              "      <td>0.164093</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>1000 rows × 4 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-79ada582-d014-4d10-837d-ca03101d0e19')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-79ada582-d014-4d10-837d-ca03101d0e19 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-79ada582-d014-4d10-837d-ca03101d0e19');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-0f95b57d-4465-45c5-bac7-e273c9731b1b\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-0f95b57d-4465-45c5-bac7-e273c9731b1b')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-0f95b57d-4465-45c5-bac7-e273c9731b1b button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "df_eval",
              "summary": "{\n  \"name\": \"df_eval\",\n  \"rows\": 1000,\n  \"fields\": [\n    {\n      \"column\": \"name\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 5,\n        \"samples\": [\n          \"PL2\",\n          \"rushton >> PL2\",\n          \"hunspell >> PL2\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"qid\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 200,\n        \"samples\": [\n          \"2864267\",\n          \"1287437\",\n          \"1623623\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"measure\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          \"ndcg_cut.10\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"value\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.20428789745765732,\n        \"min\": 0.0,\n        \"max\": 0.9765911666162366,\n        \"num_unique_values\": 301,\n        \"samples\": [\n          0.6119795246686358\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# The oracle (that always chooses the best variant out of \"no spelling correction\", \"hunspell\", \"rusthon\", \"chatGPT\")\n",
        "df_eval.groupby('qid').agg({'value': 'max'}).reset_index()['value'].mean()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Mj3ABqjw4V4w",
        "outputId": "121e8c4e-624e-4aae-dc6c-f94cfb54f13d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.5028046380905374"
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [],
      "metadata": {
        "id": "pPnUaUkP1a9x"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Statistical Analysis"
      ],
      "metadata": {
        "id": "Vq6RdTou1LHJ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def count_spelling_differences(dataset):\n",
        "  topics = pt.get_dataset(f\"irds:ir-benchmarks/{dataset}\").get_topics('query')\n",
        "  topics = {i['qid']: i['query'] for _, i in topics.iterrows()}\n",
        "\n",
        "  hunspell = tira.pt.transform_queries('workshop-on-open-web-search/qspell/hunspell', dataset)\n",
        "  topics_corrected = hunspell(pt.get_dataset(f\"irds:ir-benchmarks/{dataset}\").get_topics('query'))\n",
        "  topics_corrected = {i['qid']: i['query'] for _, i in topics_corrected.iterrows()}\n",
        "\n",
        "  pyspell = tira.pt.transform_queries('workshop-on-open-web-search/qspell/pyspell', dataset)\n",
        "  topics_corrected_pyspell = pyspell(pt.get_dataset(f\"irds:ir-benchmarks/{dataset}\").get_topics('query'))\n",
        "  topics_corrected_pyspell = {i['qid']: i['query'] for _, i in topics_corrected_pyspell.iterrows()}\n",
        "\n",
        "  rushton = tira.pt.transform_queries('workshop-on-open-web-search/qspell/rushton', dataset)\n",
        "  topics_corrected_rushton = rushton(pt.get_dataset(f\"irds:ir-benchmarks/{dataset}\").get_topics('query'))\n",
        "  topics_corrected_rushton = {i['qid']: i['query'] for _, i in topics_corrected_rushton.iterrows()}\n",
        "\n",
        "  spelling_differences_tokenized_hunspell = 0\n",
        "  spelling_differences_tokenized_pyspell = 0\n",
        "  spelling_differences_tokenized_rushton = 0\n",
        "\n",
        "  for qid in topics:\n",
        "    orignal_query = topics[qid]\n",
        "\n",
        "    spell_corrected_query = topics_corrected[qid]\n",
        "    spelling_differences_tokenized_hunspell += 0 if pt_tokenise(orignal_query).lower() == pt_tokenise(spell_corrected_query).lower() else 1\n",
        "\n",
        "    spell_corrected_query = topics_corrected_pyspell[qid]\n",
        "    spelling_differences_tokenized_pyspell += 0 if pt_tokenise(orignal_query).lower() == pt_tokenise(spell_corrected_query).lower() else 1\n",
        "\n",
        "    spell_corrected_query = topics_corrected_rushton[qid]\n",
        "    spelling_differences_tokenized_rushton += 0 if pt_tokenise(orignal_query).lower() == pt_tokenise(spell_corrected_query).lower() else 1\n",
        "\n",
        "\n",
        "  return {\n",
        "      'num_queries': len(topics),\n",
        "      'spelling_differences_hunspell': spelling_differences_tokenized_hunspell,\n",
        "      'spelling_differences_pyspell': spelling_differences_tokenized_pyspell,\n",
        "      'spelling_differences_rusthon': spelling_differences_tokenized_rushton,\n",
        "  }\n"
      ],
      "metadata": {
        "id": "AeyAQLxJP8Oi"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "datasets = [\n",
        "    'antique-test-20230107-training', 'argsme-touche-2021-task-1-20230209-training', 'argsme-touche-2020-task-1-20230209-training',\n",
        "    'clueweb09-en-trec-web-2009-20230107-training', 'clueweb09-en-trec-web-2010-20230107-training', 'clueweb09-en-trec-web-2011-20230107-training',\n",
        "    'clueweb09-en-trec-web-2012-20230107-training', 'clueweb12-touche-2020-task-2-20230209-training', 'clueweb12-touche-2021-task-2-20230209-training',\n",
        "    'clueweb12-trec-misinfo-2019-20240214-training', 'clueweb12-trec-web-2013-20230107-training', 'clueweb12-trec-web-2014-20230107-training',\n",
        "    'cord19-fulltext-trec-covid-20230107-training', 'cranfield-20230107-training', 'disks45-nocr-trec-robust-2004-20230209-training',\n",
        "    'disks45-nocr-trec7-20230209-training', 'disks45-nocr-trec8-20230209-training', 'gov-trec-web-2002-20230209-training',\n",
        "    'gov-trec-web-2003-20230209-training', 'gov-trec-web-2004-20230209-training', 'gov2-trec-tb-2004-20230209-training',\n",
        "    'gov2-trec-tb-2005-20230209-training', 'gov2-trec-tb-2006-20230209-training', 'longeval-heldout-20230513-training',\n",
        "    'longeval-long-september-20230513-training', 'longeval-short-july-20230513-training', 'longeval-train-20230513-training',\n",
        "    'medline-2004-trec-genomics-2004-20230107-training', 'medline-2004-trec-genomics-2005-20230107-training', 'medline-2017-trec-pm-2017-20230211-training',\n",
        "    'medline-2017-trec-pm-2018-20230211-training', 'msmarco-passage-trec-dl-2019-judged-20230107-training', 'msmarco-passage-trec-dl-2020-judged-20230107-training',\n",
        "    'nfcorpus-test-20230107-training', 'trec-tip-of-the-tongue-dev-20230607-training', 'vaswani-20230107-training',\n",
        "    'wapo-v2-trec-core-2018-20230107-training'\n",
        "]\n",
        "\n",
        "df = []\n",
        "for dataset in tqdm(datasets):\n",
        "  i = count_spelling_differences(dataset)\n",
        "  i['dataset']= dataset\n",
        "  df += [i]\n",
        "\n",
        "df = pd.DataFrame(df)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 211
        },
        "id": "rXpeYyrUUyHa",
        "outputId": "41df9948-78a7-4854-d6ed-013fbb6857df"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "name 'tqdm' is not defined",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-1-5045605191b1>\u001b[0m in \u001b[0;36m<cell line: 18>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     16\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 18\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mdataset\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtqdm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdatasets\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     19\u001b[0m   \u001b[0mi\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcount_spelling_differences\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m   \u001b[0mi\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'dataset'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m=\u001b[0m \u001b[0mdataset\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'tqdm' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(df[['num_queries', 'spelling_differences_rusthon']].sum())\n",
        "\n",
        "print(107/5320)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jFkhim0nYltn",
        "outputId": "bb80cfc0-5843-4cf6-d5ef-efe5f9b4392a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "num_queries                     5320\n",
            "spelling_differences_rusthon     107\n",
            "dtype: int64\n",
            "0.020112781954887218\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df['Errors in Percent (Hunspell)'] = df['spelling_differences_hunspell'] / df['num_queries']\n",
        "df['Errors in Percent (PySpell)'] = df['spelling_differences_pyspell'] / df['num_queries']\n",
        "df['Errors in Percent (Rushton)'] = df['spelling_differences_rusthon'] / df['num_queries']\n",
        "df[['dataset', 'num_queries', 'Errors in Percent (Rushton)', 'spelling_differences_rusthon',  'Errors in Percent (Hunspell)', 'spelling_differences_hunspell', 'Errors in Percent (PySpell)', 'spelling_differences_pyspell']].sort_values('Errors in Percent (Rushton)')\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "C51Ni_vOSgWm",
        "outputId": "e3603aa1-fa3d-4ed0-eb84-39d333be257c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                              dataset  num_queries  \\\n",
              "18                gov-trec-web-2003-20230209-training           50   \n",
              "33                    nfcorpus-test-20230107-training          325   \n",
              "32  msmarco-passage-trec-dl-2020-judged-20230107-t...           54   \n",
              "27  medline-2004-trec-genomics-2004-20230107-training           50   \n",
              "23                 longeval-heldout-20230513-training           98   \n",
              "22                gov2-trec-tb-2006-20230209-training           50   \n",
              "21                gov2-trec-tb-2005-20230209-training           50   \n",
              "20                gov2-trec-tb-2004-20230209-training           50   \n",
              "19                gov-trec-web-2004-20230209-training          225   \n",
              "35                          vaswani-20230107-training           93   \n",
              "17                gov-trec-web-2002-20230209-training           50   \n",
              "16               disks45-nocr-trec8-20230209-training           50   \n",
              "15               disks45-nocr-trec7-20230209-training           50   \n",
              "36           wapo-v2-trec-core-2018-20230107-training           50   \n",
              "12       cord19-fulltext-trec-covid-20230107-training           50   \n",
              "11          clueweb12-trec-web-2014-20230107-training           50   \n",
              "10          clueweb12-trec-web-2013-20230107-training           50   \n",
              "9       clueweb12-trec-misinfo-2019-20240214-training           51   \n",
              "8      clueweb12-touche-2021-task-2-20230209-training           50   \n",
              "7      clueweb12-touche-2020-task-2-20230209-training           50   \n",
              "6        clueweb09-en-trec-web-2012-20230107-training           50   \n",
              "5        clueweb09-en-trec-web-2011-20230107-training           50   \n",
              "4        clueweb09-en-trec-web-2010-20230107-training           50   \n",
              "3        clueweb09-en-trec-web-2009-20230107-training           50   \n",
              "2         argsme-touche-2020-task-1-20230209-training           49   \n",
              "14    disks45-nocr-trec-robust-2004-20230209-training          250   \n",
              "13                        cranfield-20230107-training          225   \n",
              "26                   longeval-train-20230513-training          672   \n",
              "24          longeval-long-september-20230513-training          923   \n",
              "25              longeval-short-july-20230513-training          882   \n",
              "28  medline-2004-trec-genomics-2005-20230107-training           50   \n",
              "1         argsme-touche-2021-task-1-20230209-training           50   \n",
              "31  msmarco-passage-trec-dl-2019-judged-20230107-t...           43   \n",
              "0                      antique-test-20230107-training          200   \n",
              "30        medline-2017-trec-pm-2018-20230211-training           50   \n",
              "34       trec-tip-of-the-tongue-dev-20230607-training          150   \n",
              "29        medline-2017-trec-pm-2017-20230211-training           30   \n",
              "\n",
              "    Errors in Percent (Rushton)  spelling_differences_rusthon  \\\n",
              "18                     0.000000                             0   \n",
              "33                     0.000000                             0   \n",
              "32                     0.000000                             0   \n",
              "27                     0.000000                             0   \n",
              "23                     0.000000                             0   \n",
              "22                     0.000000                             0   \n",
              "21                     0.000000                             0   \n",
              "20                     0.000000                             0   \n",
              "19                     0.000000                             0   \n",
              "35                     0.000000                             0   \n",
              "17                     0.000000                             0   \n",
              "16                     0.000000                             0   \n",
              "15                     0.000000                             0   \n",
              "36                     0.000000                             0   \n",
              "12                     0.000000                             0   \n",
              "11                     0.000000                             0   \n",
              "10                     0.000000                             0   \n",
              "9                      0.000000                             0   \n",
              "8                      0.000000                             0   \n",
              "7                      0.000000                             0   \n",
              "6                      0.000000                             0   \n",
              "5                      0.000000                             0   \n",
              "4                      0.000000                             0   \n",
              "3                      0.000000                             0   \n",
              "2                      0.000000                             0   \n",
              "14                     0.004000                             1   \n",
              "13                     0.004444                             1   \n",
              "26                     0.004464                             3   \n",
              "24                     0.006501                             6   \n",
              "25                     0.006803                             6   \n",
              "28                     0.020000                             1   \n",
              "1                      0.020000                             1   \n",
              "31                     0.023256                             1   \n",
              "0                      0.040000                             8   \n",
              "30                     0.280000                            14   \n",
              "34                     0.340000                            51   \n",
              "29                     0.466667                            14   \n",
              "\n",
              "    Errors in Percent (Hunspell)  spelling_differences_hunspell  \\\n",
              "18                      0.060000                              3   \n",
              "33                      0.332308                            108   \n",
              "32                      0.203704                             11   \n",
              "27                      0.540000                             27   \n",
              "23                      0.244898                             24   \n",
              "22                      0.060000                              3   \n",
              "21                      0.100000                              5   \n",
              "20                      0.120000                              6   \n",
              "19                      0.151111                             34   \n",
              "35                      0.064516                              6   \n",
              "17                      0.040000                              2   \n",
              "16                      0.060000                              3   \n",
              "15                      0.120000                              6   \n",
              "36                      0.120000                              6   \n",
              "12                      0.240000                             12   \n",
              "11                      0.120000                              6   \n",
              "10                      0.100000                              5   \n",
              "9                       0.117647                              6   \n",
              "8                       0.040000                              2   \n",
              "7                       0.120000                              6   \n",
              "6                       0.120000                              6   \n",
              "5                       0.260000                             13   \n",
              "4                       0.120000                              6   \n",
              "3                       0.100000                              5   \n",
              "2                       0.000000                              0   \n",
              "14                      0.088000                             22   \n",
              "13                      0.791111                            178   \n",
              "26                      0.202381                            136   \n",
              "24                      0.210184                            194   \n",
              "25                      0.221088                            195   \n",
              "28                      0.820000                             41   \n",
              "1                       0.040000                              2   \n",
              "31                      0.325581                             14   \n",
              "0                       0.320000                             64   \n",
              "30                      0.680000                             34   \n",
              "34                      0.813333                            122   \n",
              "29                      0.866667                             26   \n",
              "\n",
              "    Errors in Percent (PySpell)  spelling_differences_pyspell  \n",
              "18                     0.100000                             5  \n",
              "33                     0.298462                            97  \n",
              "32                     0.425926                            23  \n",
              "27                     0.720000                            36  \n",
              "23                     0.367347                            36  \n",
              "22                     0.360000                            18  \n",
              "21                     0.280000                            14  \n",
              "20                     0.240000                            12  \n",
              "19                     0.355556                            80  \n",
              "35                     0.096774                             9  \n",
              "17                     0.180000                             9  \n",
              "16                     0.220000                            11  \n",
              "15                     0.260000                            13  \n",
              "36                     0.260000                            13  \n",
              "12                     0.980000                            49  \n",
              "11                     0.340000                            17  \n",
              "10                     0.360000                            18  \n",
              "9                      0.176471                             9  \n",
              "8                      0.240000                            12  \n",
              "7                      0.640000                            32  \n",
              "6                      0.360000                            18  \n",
              "5                      0.560000                            28  \n",
              "4                      0.300000                            15  \n",
              "3                      0.360000                            18  \n",
              "2                      0.122449                             6  \n",
              "14                     0.304000                            76  \n",
              "13                     0.857778                           193  \n",
              "26                     0.324405                           218  \n",
              "24                     0.319610                           295  \n",
              "25                     0.344671                           304  \n",
              "28                     0.940000                            47  \n",
              "1                      0.120000                             6  \n",
              "31                     0.395349                            17  \n",
              "0                      0.455000                            91  \n",
              "30                     1.000000                            50  \n",
              "34                     0.986667                           148  \n",
              "29                     1.000000                            30  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-c2839bf2-273a-4a0d-bb49-17999c41b696\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>dataset</th>\n",
              "      <th>num_queries</th>\n",
              "      <th>Errors in Percent (Rushton)</th>\n",
              "      <th>spelling_differences_rusthon</th>\n",
              "      <th>Errors in Percent (Hunspell)</th>\n",
              "      <th>spelling_differences_hunspell</th>\n",
              "      <th>Errors in Percent (PySpell)</th>\n",
              "      <th>spelling_differences_pyspell</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>gov-trec-web-2003-20230209-training</td>\n",
              "      <td>50</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0</td>\n",
              "      <td>0.060000</td>\n",
              "      <td>3</td>\n",
              "      <td>0.100000</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>33</th>\n",
              "      <td>nfcorpus-test-20230107-training</td>\n",
              "      <td>325</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0</td>\n",
              "      <td>0.332308</td>\n",
              "      <td>108</td>\n",
              "      <td>0.298462</td>\n",
              "      <td>97</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32</th>\n",
              "      <td>msmarco-passage-trec-dl-2020-judged-20230107-t...</td>\n",
              "      <td>54</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0</td>\n",
              "      <td>0.203704</td>\n",
              "      <td>11</td>\n",
              "      <td>0.425926</td>\n",
              "      <td>23</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>27</th>\n",
              "      <td>medline-2004-trec-genomics-2004-20230107-training</td>\n",
              "      <td>50</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0</td>\n",
              "      <td>0.540000</td>\n",
              "      <td>27</td>\n",
              "      <td>0.720000</td>\n",
              "      <td>36</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23</th>\n",
              "      <td>longeval-heldout-20230513-training</td>\n",
              "      <td>98</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0</td>\n",
              "      <td>0.244898</td>\n",
              "      <td>24</td>\n",
              "      <td>0.367347</td>\n",
              "      <td>36</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22</th>\n",
              "      <td>gov2-trec-tb-2006-20230209-training</td>\n",
              "      <td>50</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0</td>\n",
              "      <td>0.060000</td>\n",
              "      <td>3</td>\n",
              "      <td>0.360000</td>\n",
              "      <td>18</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21</th>\n",
              "      <td>gov2-trec-tb-2005-20230209-training</td>\n",
              "      <td>50</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0</td>\n",
              "      <td>0.100000</td>\n",
              "      <td>5</td>\n",
              "      <td>0.280000</td>\n",
              "      <td>14</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>gov2-trec-tb-2004-20230209-training</td>\n",
              "      <td>50</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0</td>\n",
              "      <td>0.120000</td>\n",
              "      <td>6</td>\n",
              "      <td>0.240000</td>\n",
              "      <td>12</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>gov-trec-web-2004-20230209-training</td>\n",
              "      <td>225</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0</td>\n",
              "      <td>0.151111</td>\n",
              "      <td>34</td>\n",
              "      <td>0.355556</td>\n",
              "      <td>80</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35</th>\n",
              "      <td>vaswani-20230107-training</td>\n",
              "      <td>93</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0</td>\n",
              "      <td>0.064516</td>\n",
              "      <td>6</td>\n",
              "      <td>0.096774</td>\n",
              "      <td>9</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>gov-trec-web-2002-20230209-training</td>\n",
              "      <td>50</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0</td>\n",
              "      <td>0.040000</td>\n",
              "      <td>2</td>\n",
              "      <td>0.180000</td>\n",
              "      <td>9</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>disks45-nocr-trec8-20230209-training</td>\n",
              "      <td>50</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0</td>\n",
              "      <td>0.060000</td>\n",
              "      <td>3</td>\n",
              "      <td>0.220000</td>\n",
              "      <td>11</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>disks45-nocr-trec7-20230209-training</td>\n",
              "      <td>50</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0</td>\n",
              "      <td>0.120000</td>\n",
              "      <td>6</td>\n",
              "      <td>0.260000</td>\n",
              "      <td>13</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>36</th>\n",
              "      <td>wapo-v2-trec-core-2018-20230107-training</td>\n",
              "      <td>50</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0</td>\n",
              "      <td>0.120000</td>\n",
              "      <td>6</td>\n",
              "      <td>0.260000</td>\n",
              "      <td>13</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>cord19-fulltext-trec-covid-20230107-training</td>\n",
              "      <td>50</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0</td>\n",
              "      <td>0.240000</td>\n",
              "      <td>12</td>\n",
              "      <td>0.980000</td>\n",
              "      <td>49</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>clueweb12-trec-web-2014-20230107-training</td>\n",
              "      <td>50</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0</td>\n",
              "      <td>0.120000</td>\n",
              "      <td>6</td>\n",
              "      <td>0.340000</td>\n",
              "      <td>17</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>clueweb12-trec-web-2013-20230107-training</td>\n",
              "      <td>50</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0</td>\n",
              "      <td>0.100000</td>\n",
              "      <td>5</td>\n",
              "      <td>0.360000</td>\n",
              "      <td>18</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>clueweb12-trec-misinfo-2019-20240214-training</td>\n",
              "      <td>51</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0</td>\n",
              "      <td>0.117647</td>\n",
              "      <td>6</td>\n",
              "      <td>0.176471</td>\n",
              "      <td>9</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>clueweb12-touche-2021-task-2-20230209-training</td>\n",
              "      <td>50</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0</td>\n",
              "      <td>0.040000</td>\n",
              "      <td>2</td>\n",
              "      <td>0.240000</td>\n",
              "      <td>12</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>clueweb12-touche-2020-task-2-20230209-training</td>\n",
              "      <td>50</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0</td>\n",
              "      <td>0.120000</td>\n",
              "      <td>6</td>\n",
              "      <td>0.640000</td>\n",
              "      <td>32</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>clueweb09-en-trec-web-2012-20230107-training</td>\n",
              "      <td>50</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0</td>\n",
              "      <td>0.120000</td>\n",
              "      <td>6</td>\n",
              "      <td>0.360000</td>\n",
              "      <td>18</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>clueweb09-en-trec-web-2011-20230107-training</td>\n",
              "      <td>50</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0</td>\n",
              "      <td>0.260000</td>\n",
              "      <td>13</td>\n",
              "      <td>0.560000</td>\n",
              "      <td>28</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>clueweb09-en-trec-web-2010-20230107-training</td>\n",
              "      <td>50</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0</td>\n",
              "      <td>0.120000</td>\n",
              "      <td>6</td>\n",
              "      <td>0.300000</td>\n",
              "      <td>15</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>clueweb09-en-trec-web-2009-20230107-training</td>\n",
              "      <td>50</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0</td>\n",
              "      <td>0.100000</td>\n",
              "      <td>5</td>\n",
              "      <td>0.360000</td>\n",
              "      <td>18</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>argsme-touche-2020-task-1-20230209-training</td>\n",
              "      <td>49</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0</td>\n",
              "      <td>0.122449</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>disks45-nocr-trec-robust-2004-20230209-training</td>\n",
              "      <td>250</td>\n",
              "      <td>0.004000</td>\n",
              "      <td>1</td>\n",
              "      <td>0.088000</td>\n",
              "      <td>22</td>\n",
              "      <td>0.304000</td>\n",
              "      <td>76</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>cranfield-20230107-training</td>\n",
              "      <td>225</td>\n",
              "      <td>0.004444</td>\n",
              "      <td>1</td>\n",
              "      <td>0.791111</td>\n",
              "      <td>178</td>\n",
              "      <td>0.857778</td>\n",
              "      <td>193</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26</th>\n",
              "      <td>longeval-train-20230513-training</td>\n",
              "      <td>672</td>\n",
              "      <td>0.004464</td>\n",
              "      <td>3</td>\n",
              "      <td>0.202381</td>\n",
              "      <td>136</td>\n",
              "      <td>0.324405</td>\n",
              "      <td>218</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24</th>\n",
              "      <td>longeval-long-september-20230513-training</td>\n",
              "      <td>923</td>\n",
              "      <td>0.006501</td>\n",
              "      <td>6</td>\n",
              "      <td>0.210184</td>\n",
              "      <td>194</td>\n",
              "      <td>0.319610</td>\n",
              "      <td>295</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25</th>\n",
              "      <td>longeval-short-july-20230513-training</td>\n",
              "      <td>882</td>\n",
              "      <td>0.006803</td>\n",
              "      <td>6</td>\n",
              "      <td>0.221088</td>\n",
              "      <td>195</td>\n",
              "      <td>0.344671</td>\n",
              "      <td>304</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>28</th>\n",
              "      <td>medline-2004-trec-genomics-2005-20230107-training</td>\n",
              "      <td>50</td>\n",
              "      <td>0.020000</td>\n",
              "      <td>1</td>\n",
              "      <td>0.820000</td>\n",
              "      <td>41</td>\n",
              "      <td>0.940000</td>\n",
              "      <td>47</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>argsme-touche-2021-task-1-20230209-training</td>\n",
              "      <td>50</td>\n",
              "      <td>0.020000</td>\n",
              "      <td>1</td>\n",
              "      <td>0.040000</td>\n",
              "      <td>2</td>\n",
              "      <td>0.120000</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>31</th>\n",
              "      <td>msmarco-passage-trec-dl-2019-judged-20230107-t...</td>\n",
              "      <td>43</td>\n",
              "      <td>0.023256</td>\n",
              "      <td>1</td>\n",
              "      <td>0.325581</td>\n",
              "      <td>14</td>\n",
              "      <td>0.395349</td>\n",
              "      <td>17</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>antique-test-20230107-training</td>\n",
              "      <td>200</td>\n",
              "      <td>0.040000</td>\n",
              "      <td>8</td>\n",
              "      <td>0.320000</td>\n",
              "      <td>64</td>\n",
              "      <td>0.455000</td>\n",
              "      <td>91</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>30</th>\n",
              "      <td>medline-2017-trec-pm-2018-20230211-training</td>\n",
              "      <td>50</td>\n",
              "      <td>0.280000</td>\n",
              "      <td>14</td>\n",
              "      <td>0.680000</td>\n",
              "      <td>34</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>50</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>34</th>\n",
              "      <td>trec-tip-of-the-tongue-dev-20230607-training</td>\n",
              "      <td>150</td>\n",
              "      <td>0.340000</td>\n",
              "      <td>51</td>\n",
              "      <td>0.813333</td>\n",
              "      <td>122</td>\n",
              "      <td>0.986667</td>\n",
              "      <td>148</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>29</th>\n",
              "      <td>medline-2017-trec-pm-2017-20230211-training</td>\n",
              "      <td>30</td>\n",
              "      <td>0.466667</td>\n",
              "      <td>14</td>\n",
              "      <td>0.866667</td>\n",
              "      <td>26</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>30</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-c2839bf2-273a-4a0d-bb49-17999c41b696')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-c2839bf2-273a-4a0d-bb49-17999c41b696 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-c2839bf2-273a-4a0d-bb49-17999c41b696');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-7b3ce4b4-599f-4012-a299-229b5ac860b3\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-7b3ce4b4-599f-4012-a299-229b5ac860b3')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-7b3ce4b4-599f-4012-a299-229b5ac860b3 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"df[['dataset', 'num_queries', 'Errors in Percent (Rushton)', 'spelling_differences_rusthon',  'Errors in Percent (Hunspell)', 'spelling_differences_hunspell', 'Errors in Percent (PySpell)', 'spelling_differences_pyspell']]\",\n  \"rows\": 37,\n  \"fields\": [\n    {\n      \"column\": \"dataset\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 37,\n        \"samples\": [\n          \"clueweb12-trec-misinfo-2019-20240214-training\",\n          \"wapo-v2-trec-core-2018-20230107-training\",\n          \"longeval-heldout-20230513-training\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"num_queries\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 219,\n        \"min\": 30,\n        \"max\": 923,\n        \"num_unique_values\": 16,\n        \"samples\": [\n          50,\n          325,\n          93\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Errors in Percent (Rushton)\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.10204714495626632,\n        \"min\": 0.0,\n        \"max\": 0.4666666666666667,\n        \"num_unique_values\": 12,\n        \"samples\": [\n          0.34,\n          0.28,\n          0.0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"spelling_differences_rusthon\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 8,\n        \"min\": 0,\n        \"max\": 51,\n        \"num_unique_values\": 7,\n        \"samples\": [\n          0,\n          1,\n          14\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Errors in Percent (Hunspell)\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.24624340298935055,\n        \"min\": 0.0,\n        \"max\": 0.8666666666666667,\n        \"num_unique_values\": 25,\n        \"samples\": [\n          0.06451612903225806,\n          0.20238095238095238,\n          0.06\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"spelling_differences_hunspell\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 56,\n        \"min\": 0,\n        \"max\": 195,\n        \"num_unique_values\": 22,\n        \"samples\": [\n          3,\n          178,\n          2\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Errors in Percent (PySpell)\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.27397590505741676,\n        \"min\": 0.0967741935483871,\n        \"max\": 1.0,\n        \"num_unique_values\": 31,\n        \"samples\": [\n          0.3953488372093023,\n          0.17647058823529413,\n          0.3196099674972914\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"spelling_differences_pyspell\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 77,\n        \"min\": 5,\n        \"max\": 304,\n        \"num_unique_values\": 27,\n        \"samples\": [\n          9,\n          32,\n          11\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "count_spelling_differences()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uRlKmBv9UYFN",
        "outputId": "6cb11e75-faca-4e2d-8ea4-3d141e57115d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "There are multiple query fields available: ('text', 'title', 'query', 'description', 'narrative'). To use with pyterrier, provide variant or modify dataframe to add query column.\n",
            "There are multiple query fields available: ('text', 'title', 'query', 'description', 'narrative'). To use with pyterrier, provide variant or modify dataframe to add query column.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'num_queries': 49,\n",
              " 'spelling_differences': 49,\n",
              " 'spelling_differences_after_tokenization': 0}"
            ]
          },
          "metadata": {},
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df = dataset.get_topics()\n",
        "df['orig_query'] = df['query'].apply(lambda i: ' '.join(tokeniser.getTokens(i)))\n",
        "df = hunspell(df)\n",
        "del df['query_0']\n",
        "df[df['orig_query'] != df['query']]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "J4tZjELmGMFM",
        "outputId": "a0431db4-d601-4d39-e559-d2f507c2ee42"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "         qid                                         orig_query  \\\n",
              "0    3990512           how can we get concentration onsomething   \n",
              "2    2528767  how do i determine the charge of the iron ion ...   \n",
              "4    1880028   what does see leaflet mean on ept pregnancy test   \n",
              "6    1287437                  how can i lose 30 pounds by june3   \n",
              "7    2956570  what are the words to write the sound of raind...   \n",
              "..       ...                                                ...   \n",
              "189  1509982          how is th chemistry is a basic of science   \n",
              "194  1850323  why some men never wash their hands after usin...   \n",
              "197  1582877  why do we go to school if in the real world we...   \n",
              "198  1340574  why do some people only go to church on easter...   \n",
              "199  1971899                                  what is masturbat   \n",
              "\n",
              "                                                 query  \n",
              "0               how can we get concentration something  \n",
              "2    how do i determine the charge of the iron ion ...  \n",
              "4      what does see leaflet mean on ep pregnancy test  \n",
              "6                   how can i lose 30 pounds by jejune  \n",
              "7    what are the words to write the sound of raind...  \n",
              "..                                                 ...  \n",
              "189          how is ht chemistry is a basic of science  \n",
              "194  why some men never wash their hands after usin...  \n",
              "197  why do we go to school if in the real world we...  \n",
              "198  why do some people only go to church on easter...  \n",
              "199                                what is masturbator  \n",
              "\n",
              "[64 rows x 3 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-42f5957f-aac7-435b-9517-ba371465855b\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>qid</th>\n",
              "      <th>orig_query</th>\n",
              "      <th>query</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>3990512</td>\n",
              "      <td>how can we get concentration onsomething</td>\n",
              "      <td>how can we get concentration something</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2528767</td>\n",
              "      <td>how do i determine the charge of the iron ion ...</td>\n",
              "      <td>how do i determine the charge of the iron ion ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1880028</td>\n",
              "      <td>what does see leaflet mean on ept pregnancy test</td>\n",
              "      <td>what does see leaflet mean on ep pregnancy test</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>1287437</td>\n",
              "      <td>how can i lose 30 pounds by june3</td>\n",
              "      <td>how can i lose 30 pounds by jejune</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>2956570</td>\n",
              "      <td>what are the words to write the sound of raind...</td>\n",
              "      <td>what are the words to write the sound of raind...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>189</th>\n",
              "      <td>1509982</td>\n",
              "      <td>how is th chemistry is a basic of science</td>\n",
              "      <td>how is ht chemistry is a basic of science</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>194</th>\n",
              "      <td>1850323</td>\n",
              "      <td>why some men never wash their hands after usin...</td>\n",
              "      <td>why some men never wash their hands after usin...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>197</th>\n",
              "      <td>1582877</td>\n",
              "      <td>why do we go to school if in the real world we...</td>\n",
              "      <td>why do we go to school if in the real world we...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>198</th>\n",
              "      <td>1340574</td>\n",
              "      <td>why do some people only go to church on easter...</td>\n",
              "      <td>why do some people only go to church on easter...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>199</th>\n",
              "      <td>1971899</td>\n",
              "      <td>what is masturbat</td>\n",
              "      <td>what is masturbator</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>64 rows × 3 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-42f5957f-aac7-435b-9517-ba371465855b')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-42f5957f-aac7-435b-9517-ba371465855b button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-42f5957f-aac7-435b-9517-ba371465855b');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-d2a17f83-3153-4c34-a576-bb492a0f8448\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-d2a17f83-3153-4c34-a576-bb492a0f8448')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-d2a17f83-3153-4c34-a576-bb492a0f8448 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"df[df['orig_query'] != df['query']]\",\n  \"rows\": 64,\n  \"fields\": [\n    {\n      \"column\": \"qid\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 64,\n        \"samples\": [\n          \"2722241\",\n          \"4473137\",\n          \"3990512\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"orig_query\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 64,\n        \"samples\": [\n          \"why do some animals hide in a dark place b4 they die\",\n          \"causes and effects of the rise of christanity\",\n          \"how can we get concentration onsomething\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"query\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 64,\n        \"samples\": [\n          \"why do some animals hide in a dark place b they die\",\n          \"causes and effects of the rise of christianity\",\n          \"how can we get concentration something\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 65
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "VxI2tPtxHRDS",
        "outputId": "bdf725ec-b1a8-4e2f-a907-05ca5cf67e6f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "         qid                                         orig_query  \\\n",
              "0    3990512           how can we get concentration onsomething   \n",
              "2    2528767  how do i determine the charge of the iron ion ...   \n",
              "4    1880028   what does see leaflet mean on ept pregnancy test   \n",
              "6    1287437                  how can i lose 30 pounds by june3   \n",
              "7    2956570  what are the words to write the sound of raind...   \n",
              "..       ...                                                ...   \n",
              "189  1509982          how is th chemistry is a basic of science   \n",
              "194  1850323  why some men never wash their hands after usin...   \n",
              "197  1582877  why do we go to school if in the real world we...   \n",
              "198  1340574  why do some people only go to church on easter...   \n",
              "199  1971899                                  what is masturbat   \n",
              "\n",
              "                                                 query  \n",
              "0               how can we get concentration something  \n",
              "2    how do i determine the charge of the iron ion ...  \n",
              "4      what does see leaflet mean on ep pregnancy test  \n",
              "6                   how can i lose 30 pounds by jejune  \n",
              "7    what are the words to write the sound of raind...  \n",
              "..                                                 ...  \n",
              "189          how is ht chemistry is a basic of science  \n",
              "194  why some men never wash their hands after usin...  \n",
              "197  why do we go to school if in the real world we...  \n",
              "198  why do some people only go to church on easter...  \n",
              "199                                what is masturbator  \n",
              "\n",
              "[64 rows x 3 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-2dba9ce9-c258-4d9e-b12c-c125ebc2a76b\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>qid</th>\n",
              "      <th>orig_query</th>\n",
              "      <th>query</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>3990512</td>\n",
              "      <td>how can we get concentration onsomething</td>\n",
              "      <td>how can we get concentration something</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2528767</td>\n",
              "      <td>how do i determine the charge of the iron ion ...</td>\n",
              "      <td>how do i determine the charge of the iron ion ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1880028</td>\n",
              "      <td>what does see leaflet mean on ept pregnancy test</td>\n",
              "      <td>what does see leaflet mean on ep pregnancy test</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>1287437</td>\n",
              "      <td>how can i lose 30 pounds by june3</td>\n",
              "      <td>how can i lose 30 pounds by jejune</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>2956570</td>\n",
              "      <td>what are the words to write the sound of raind...</td>\n",
              "      <td>what are the words to write the sound of raind...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>189</th>\n",
              "      <td>1509982</td>\n",
              "      <td>how is th chemistry is a basic of science</td>\n",
              "      <td>how is ht chemistry is a basic of science</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>194</th>\n",
              "      <td>1850323</td>\n",
              "      <td>why some men never wash their hands after usin...</td>\n",
              "      <td>why some men never wash their hands after usin...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>197</th>\n",
              "      <td>1582877</td>\n",
              "      <td>why do we go to school if in the real world we...</td>\n",
              "      <td>why do we go to school if in the real world we...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>198</th>\n",
              "      <td>1340574</td>\n",
              "      <td>why do some people only go to church on easter...</td>\n",
              "      <td>why do some people only go to church on easter...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>199</th>\n",
              "      <td>1971899</td>\n",
              "      <td>what is masturbat</td>\n",
              "      <td>what is masturbator</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>64 rows × 3 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-2dba9ce9-c258-4d9e-b12c-c125ebc2a76b')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-2dba9ce9-c258-4d9e-b12c-c125ebc2a76b button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-2dba9ce9-c258-4d9e-b12c-c125ebc2a76b');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-dcd92403-89fd-401f-bbd9-56f7985efd1f\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-dcd92403-89fd-401f-bbd9-56f7985efd1f')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-dcd92403-89fd-401f-bbd9-56f7985efd1f button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"df[df['orig_query'] != df['query']]\",\n  \"rows\": 64,\n  \"fields\": [\n    {\n      \"column\": \"qid\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 64,\n        \"samples\": [\n          \"2722241\",\n          \"4473137\",\n          \"3990512\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"orig_query\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 64,\n        \"samples\": [\n          \"why do some animals hide in a dark place b4 they die\",\n          \"causes and effects of the rise of christanity\",\n          \"how can we get concentration onsomething\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"query\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 64,\n        \"samples\": [\n          \"why do some animals hide in a dark place b they die\",\n          \"causes and effects of the rise of christianity\",\n          \"how can we get concentration something\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 64
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "(dataset.get_topics()['query'] == hunspell(dataset.get_topics())['query']).value_counts()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IeVxHnjz_Q9z",
        "outputId": "91b0df34-0c82-4d7a-cdfa-4ae906fbbcad"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True     136\n",
              "False     64\n",
              "Name: query, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 56
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "dataset.get_topics()['query'].iloc[1]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "WFNrP59p7IGf",
        "outputId": "b5e3d332-d5d2-477b-9197-91be99c69c2f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'why doesn t the water fall off earth if it s round'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 51
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "HpvM_YRiFsys",
        "outputId": "faf98dd5-ab4a-4c94-8ea9-235d7b265703"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'why doesn t the water fall off earth if it s round'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 53
        }
      ]
    }
  ]
}