Disfluency_Detection_Model.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "gpuClass": "standard",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "4e9fdbbecf144aca84af38ae052a9c21": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_2ddefd0514c84487a6e8014266e09ab1",
              "IPY_MODEL_7033ac4552b24f3db3566396a3ada30e",
              "IPY_MODEL_20393b312c3b4f80bd9974cdd6de6a57"
            ],
            "layout": "IPY_MODEL_3e6a7c2a7b754494aa42decc922f8cac"
          }
        },
        "2ddefd0514c84487a6e8014266e09ab1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3f66ecafc4794a4396f488333b66931d",
            "placeholder": "​",
            "style": "IPY_MODEL_2832d33b924540868c2fea13e9a24f4e",
            "value": "Downloading: 100%"
          }
        },
        "7033ac4552b24f3db3566396a3ada30e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a9063ea34fae4237a2defbb809f832fd",
            "max": 28,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_986babfb154d459f883343a8a125ba90",
            "value": 28
          }
        },
        "20393b312c3b4f80bd9974cdd6de6a57": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_31e3b629cdf644d89f648c895a558a95",
            "placeholder": "​",
            "style": "IPY_MODEL_6129c4ed8944451ab40cfb5db52e592a",
            "value": " 28.0/28.0 [00:00&lt;00:00, 365B/s]"
          }
        },
        "3e6a7c2a7b754494aa42decc922f8cac": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3f66ecafc4794a4396f488333b66931d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2832d33b924540868c2fea13e9a24f4e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "a9063ea34fae4237a2defbb809f832fd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "986babfb154d459f883343a8a125ba90": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "31e3b629cdf644d89f648c895a558a95": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6129c4ed8944451ab40cfb5db52e592a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8fb5dbddce094de098a13713dd508ff0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_7a29931518d2495da197329e3b3a29a3",
              "IPY_MODEL_bbaa16bf555341e89b7eb12eee42b4c0",
              "IPY_MODEL_e7447f907b6c43288c6aa783b5de6d7d"
            ],
            "layout": "IPY_MODEL_214b6b59bb0b4ed1a871cb4c531c3da7"
          }
        },
        "7a29931518d2495da197329e3b3a29a3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ea43d255f38c47dd834ebfec72b36fa6",
            "placeholder": "​",
            "style": "IPY_MODEL_3eeef0fa9d71455d837b7e561600591e",
            "value": "Downloading: 100%"
          }
        },
        "bbaa16bf555341e89b7eb12eee42b4c0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fe37108810cd46cb9bad2ad869641e4e",
            "max": 483,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_d7eb8f19c6e14fe8bc688bac2c5a1646",
            "value": 483
          }
        },
        "e7447f907b6c43288c6aa783b5de6d7d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cfc4f9daa1c64c10b7b658eeb1f90ed0",
            "placeholder": "​",
            "style": "IPY_MODEL_f9898223aeea4665856e162d0ae6c59f",
            "value": " 483/483 [00:00&lt;00:00, 6.43kB/s]"
          }
        },
        "214b6b59bb0b4ed1a871cb4c531c3da7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ea43d255f38c47dd834ebfec72b36fa6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3eeef0fa9d71455d837b7e561600591e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "fe37108810cd46cb9bad2ad869641e4e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d7eb8f19c6e14fe8bc688bac2c5a1646": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "cfc4f9daa1c64c10b7b658eeb1f90ed0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f9898223aeea4665856e162d0ae6c59f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "398774bdb22f4160acd0c34537e6f2a5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_908fc2d0fc8c4c4b8213bbb320dba612",
              "IPY_MODEL_8992f3272951416aa4d9d6b0d8d6308d",
              "IPY_MODEL_1a6ba27915624faf816133c0a82ea800"
            ],
            "layout": "IPY_MODEL_e2c23a48c58f4660ad04316a172343e1"
          }
        },
        "908fc2d0fc8c4c4b8213bbb320dba612": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9ec4a273114d4fff8a2909a80210b2a7",
            "placeholder": "​",
            "style": "IPY_MODEL_893c89b6b2aa4a2eb998c4ca40c763ca",
            "value": "Downloading: 100%"
          }
        },
        "8992f3272951416aa4d9d6b0d8d6308d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_78fc985159134280bf9d5c587a3beb8d",
            "max": 231508,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_b35d13bdeb4a4139ac592a5e45c9d173",
            "value": 231508
          }
        },
        "1a6ba27915624faf816133c0a82ea800": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e1d3742130c24d129ca0600bf246a63f",
            "placeholder": "​",
            "style": "IPY_MODEL_9a33c0a344f04398a084b2887deadb36",
            "value": " 232k/232k [00:00&lt;00:00, 2.56MB/s]"
          }
        },
        "e2c23a48c58f4660ad04316a172343e1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9ec4a273114d4fff8a2909a80210b2a7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "893c89b6b2aa4a2eb998c4ca40c763ca": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "78fc985159134280bf9d5c587a3beb8d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b35d13bdeb4a4139ac592a5e45c9d173": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "e1d3742130c24d129ca0600bf246a63f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9a33c0a344f04398a084b2887deadb36": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7dbc46b2f79f44da86c94029aec234d2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_3e0554557d3d43d58d4307d93851a330",
              "IPY_MODEL_e326bee990a048788b30ed6e53268abb",
              "IPY_MODEL_6bf8533fe35b4ba9b7ac99f5b943acae"
            ],
            "layout": "IPY_MODEL_55f69c54fff445e4a35e362638717235"
          }
        },
        "3e0554557d3d43d58d4307d93851a330": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8a6224cb72f84cde8b0e43d503852b42",
            "placeholder": "​",
            "style": "IPY_MODEL_93ea24773200455288bb18b2c7d8957e",
            "value": "Downloading: 100%"
          }
        },
        "e326bee990a048788b30ed6e53268abb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_55f703c630cd4206a1637e43ebe928ca",
            "max": 466062,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_6659dd86bf364c1485ad9180b5f7986c",
            "value": 466062
          }
        },
        "6bf8533fe35b4ba9b7ac99f5b943acae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a2626bc8e35643789807f60ec1def0b1",
            "placeholder": "​",
            "style": "IPY_MODEL_996de5d6ce744144b8270ece86785c1c",
            "value": " 466k/466k [00:00&lt;00:00, 2.67MB/s]"
          }
        },
        "55f69c54fff445e4a35e362638717235": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8a6224cb72f84cde8b0e43d503852b42": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "93ea24773200455288bb18b2c7d8957e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "55f703c630cd4206a1637e43ebe928ca": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6659dd86bf364c1485ad9180b5f7986c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "a2626bc8e35643789807f60ec1def0b1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "996de5d6ce744144b8270ece86785c1c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "482ac6fa4d9a492ba6a5523e57a31d42": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_a264a29dfa6d40f09b00b170f851fc0d",
              "IPY_MODEL_e32f8c7c25e7412493ba27ddcfe8eb70",
              "IPY_MODEL_918c082ee0f64447bdc8b9b0eeaa1dfb"
            ],
            "layout": "IPY_MODEL_101c8dd4de434f0384cbb365e474292f"
          }
        },
        "a264a29dfa6d40f09b00b170f851fc0d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_97f3c6d599e34b83a6a80983016309bf",
            "placeholder": "​",
            "style": "IPY_MODEL_2fca42aa96114f0f8957ac9d710c62cb",
            "value": "100%"
          }
        },
        "e32f8c7c25e7412493ba27ddcfe8eb70": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b615c3f3496e4a499ac1b78cfaffe227",
            "max": 131,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c06cf37e3ed448ecaff47c2cc43bfac9",
            "value": 131
          }
        },
        "918c082ee0f64447bdc8b9b0eeaa1dfb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7ddd3f3194574ac5a4c5ceb2ae030a79",
            "placeholder": "​",
            "style": "IPY_MODEL_74b433090de042bbbd183228aa2e3cc5",
            "value": " 131/131 [00:12&lt;00:00, 11.54ba/s]"
          }
        },
        "101c8dd4de434f0384cbb365e474292f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "97f3c6d599e34b83a6a80983016309bf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2fca42aa96114f0f8957ac9d710c62cb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b615c3f3496e4a499ac1b78cfaffe227": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c06cf37e3ed448ecaff47c2cc43bfac9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "7ddd3f3194574ac5a4c5ceb2ae030a79": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "74b433090de042bbbd183228aa2e3cc5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8e1a5990c3e943a889cef35334602b9d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_60aa9d535bd34c12bc7149f2aaf0f5f8",
              "IPY_MODEL_affa06d53b0c44d9b150936662feb51d",
              "IPY_MODEL_0058dbdd1e694211a7a75154d7691604"
            ],
            "layout": "IPY_MODEL_08b1e9027e0d4998abcfb2fe18bb6258"
          }
        },
        "60aa9d535bd34c12bc7149f2aaf0f5f8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d45b00dc5e864d5a8292a8818c3def5f",
            "placeholder": "​",
            "style": "IPY_MODEL_de64b62c69ef44718e3322bc856ceab9",
            "value": "100%"
          }
        },
        "affa06d53b0c44d9b150936662feb51d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_eb7a54a43ad74bd99957b6fd84e07757",
            "max": 44,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c55ec19133e648be8418ab54c36b52e4",
            "value": 44
          }
        },
        "0058dbdd1e694211a7a75154d7691604": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4b1bdd96978f4789b01b8e52b52f20e5",
            "placeholder": "​",
            "style": "IPY_MODEL_331fdc0e7a084ee983d334604623a05e",
            "value": " 44/44 [00:06&lt;00:00,  6.74ba/s]"
          }
        },
        "08b1e9027e0d4998abcfb2fe18bb6258": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d45b00dc5e864d5a8292a8818c3def5f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "de64b62c69ef44718e3322bc856ceab9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "eb7a54a43ad74bd99957b6fd84e07757": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c55ec19133e648be8418ab54c36b52e4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "4b1bdd96978f4789b01b8e52b52f20e5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "331fdc0e7a084ee983d334604623a05e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e6cdb0252bd7423da90d66b8e792e81e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "VBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "VBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "VBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_08c37914eba7475a9970d19d9ccf2775",
              "IPY_MODEL_90019d1d25f14c378ade0511a9ebaf25",
              "IPY_MODEL_db41f900658344ee83907a6312920554",
              "IPY_MODEL_4920a5b7c981417a986c5f89fd51740c",
              "IPY_MODEL_fda380adb4e24d0da76ab3e2470272b1"
            ],
            "layout": "IPY_MODEL_53923d483cea46cab5475e7fe1b7412b"
          }
        },
        "08c37914eba7475a9970d19d9ccf2775": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_44cf7477ab1e4c7597d3d3f9390acbff",
            "placeholder": "​",
            "style": "IPY_MODEL_731ee20423bf43d691488dec048ea8d8",
            "value": "<center> <img\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.svg\nalt='Hugging Face'> <br> Copy a token from <a\nhref=\"https://huggingface.co/settings/tokens\" target=\"_blank\">your Hugging Face\ntokens page</a> and paste it below. <br> Immediately click login after copying\nyour token or it might be stored in plain text in this notebook file. </center>"
          }
        },
        "90019d1d25f14c378ade0511a9ebaf25": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "PasswordModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "PasswordModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "PasswordView",
            "continuous_update": true,
            "description": "Token:",
            "description_tooltip": null,
            "disabled": false,
            "layout": "IPY_MODEL_f33de199c5fd435395a4689bc04f06fe",
            "placeholder": "​",
            "style": "IPY_MODEL_90e3ff69547d4409a9bdfb983aa8be51",
            "value": "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"
          }
        },
        "db41f900658344ee83907a6312920554": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "CheckboxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "CheckboxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "CheckboxView",
            "description": "Add token as git credential?",
            "description_tooltip": null,
            "disabled": false,
            "indent": true,
            "layout": "IPY_MODEL_3cffbadafafd4628909ca58d839818d2",
            "style": "IPY_MODEL_41c2a366d94b47359d6f951f037bd043",
            "value": true
          }
        },
        "4920a5b7c981417a986c5f89fd51740c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ButtonModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ButtonModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ButtonView",
            "button_style": "",
            "description": "Login",
            "disabled": false,
            "icon": "",
            "layout": "IPY_MODEL_2fe2e876b8d341ffa7a7a4d86826c8c2",
            "style": "IPY_MODEL_72cf14297fe04d3499b1a6b2cbdb2327",
            "tooltip": ""
          }
        },
        "fda380adb4e24d0da76ab3e2470272b1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_39405232588841ffb7e4dc7de1cd861c",
            "placeholder": "​",
            "style": "IPY_MODEL_e1212d03d77d42659e91f2923f6248e7",
            "value": "\n<b>Pro Tip:</b> If you don't already have one, you can create a dedicated\n'notebooks' token with 'write' access, that you can then easily reuse for all\nnotebooks. </center>"
          }
        },
        "53923d483cea46cab5475e7fe1b7412b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": "center",
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": "flex",
            "flex": null,
            "flex_flow": "column",
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "44cf7477ab1e4c7597d3d3f9390acbff": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "731ee20423bf43d691488dec048ea8d8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f33de199c5fd435395a4689bc04f06fe": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "90e3ff69547d4409a9bdfb983aa8be51": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3cffbadafafd4628909ca58d839818d2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "41c2a366d94b47359d6f951f037bd043": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2fe2e876b8d341ffa7a7a4d86826c8c2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "72cf14297fe04d3499b1a6b2cbdb2327": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ButtonStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ButtonStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "button_color": null,
            "font_weight": ""
          }
        },
        "39405232588841ffb7e4dc7de1cd861c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e1212d03d77d42659e91f2923f6248e7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "EI10pwQsP1p0"
      },
      "outputs": [],
      "source": [
        "import pandas as pd \n",
        "import numpy as np \n",
        "import matplotlib.pyplot as plt \n",
        "import seaborn \n",
        "from sklearn.model_selection import train_test_split\n",
        "import tensorflow as tf"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "whAPOVvyCwUP",
        "outputId": "470dde6a-fdac-4270-9354-e215f000ee59"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sb = pd.read_csv(\"/content/drive/MyDrive/Speech_analyzer/switchboard_corrected_with_silver_reannotation.tsv\", sep = '\\t')"
      ],
      "metadata": {
        "id": "JnS7VGKrQUna"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sb.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mKtIeeUSQnND",
        "outputId": "2e276137-9d89-42f5-a71a-4c5774ec3c08"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(193085, 12)"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sb.iloc[22]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ywh_olXIQog8",
        "outputId": "b8d53daf-99b5-4b81-f63f-0e464ed5631d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "speaker                       A\n",
              "turn                         11\n",
              "sent_num                      0\n",
              "file               sw2005.trans\n",
              "sentence               ['yeah']\n",
              "ms_sentence                  []\n",
              "comb_sentence          ['yeah']\n",
              "names            ['uwA11_pw33']\n",
              "ms_names                     []\n",
              "comb_ann                ['INS']\n",
              "first_name           uwA11_pw33\n",
              "ms_disfl                     []\n",
              "Name: 22, dtype: object"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Data Preprocessing"
      ],
      "metadata": {
        "id": "rG5Xyu4qGL2E"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "sb = sb[['ms_sentence','ms_disfl']]"
      ],
      "metadata": {
        "id": "VtLCs939QzMM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sb"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "ErEn1Z2pQzjy",
        "outputId": "5731abde-c279-436d-8fc1-ce64cd1abac9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                              ms_sentence  \\\n",
              "0                                                ['okay']   \n",
              "1       ['uh', 'first', 'um', 'i', 'need', 'to', 'know...   \n",
              "2       ['well', 'of', 'course', 'it', \"'s\", 'you', 'k...   \n",
              "3                                                 ['yes']   \n",
              "4                                                ['yeah']   \n",
              "...                                                   ...   \n",
              "193080  ['cause', 'i', 'have', \"n't\", 'been', 'there',...   \n",
              "193081  ['well', 'you', 'know', 'y-', 'going', 'over',...   \n",
              "193082                                           ['yeah']   \n",
              "193083  ['you', 'can', 'see', 'that', 'horrible', 'hor...   \n",
              "193084  ['well', 'we', 'were', 'there', 'when', 'they'...   \n",
              "\n",
              "                                                 ms_disfl  \n",
              "0                                                   ['O']  \n",
              "1       ['O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'BE_I...  \n",
              "2       ['O', 'O', 'O', 'BE', 'IP', 'C', 'C', 'C', 'C'...  \n",
              "3                                                   ['O']  \n",
              "4                                                   ['O']  \n",
              "...                                                   ...  \n",
              "193080                ['O', 'O', 'O', 'O', 'O', 'O', 'O']  \n",
              "193081  ['O', 'O', 'O', 'BE_IP', 'O', 'O', 'O', 'O', '...  \n",
              "193082                                              ['O']  \n",
              "193083           ['O', 'O', 'O', 'O', 'O', 'O', 'O', 'O']  \n",
              "193084           ['O', 'O', 'O', 'O', 'O', 'O', 'O', 'O']  \n",
              "\n",
              "[193085 rows x 2 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-8feab487-c3de-4881-9489-e3efe08168c1\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>ms_sentence</th>\n",
              "      <th>ms_disfl</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>['okay']</td>\n",
              "      <td>['O']</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>['uh', 'first', 'um', 'i', 'need', 'to', 'know...</td>\n",
              "      <td>['O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'BE_I...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>['well', 'of', 'course', 'it', \"'s\", 'you', 'k...</td>\n",
              "      <td>['O', 'O', 'O', 'BE', 'IP', 'C', 'C', 'C', 'C'...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>['yes']</td>\n",
              "      <td>['O']</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>['yeah']</td>\n",
              "      <td>['O']</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>193080</th>\n",
              "      <td>['cause', 'i', 'have', \"n't\", 'been', 'there',...</td>\n",
              "      <td>['O', 'O', 'O', 'O', 'O', 'O', 'O']</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>193081</th>\n",
              "      <td>['well', 'you', 'know', 'y-', 'going', 'over',...</td>\n",
              "      <td>['O', 'O', 'O', 'BE_IP', 'O', 'O', 'O', 'O', '...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>193082</th>\n",
              "      <td>['yeah']</td>\n",
              "      <td>['O']</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>193083</th>\n",
              "      <td>['you', 'can', 'see', 'that', 'horrible', 'hor...</td>\n",
              "      <td>['O', 'O', 'O', 'O', 'O', 'O', 'O', 'O']</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>193084</th>\n",
              "      <td>['well', 'we', 'were', 'there', 'when', 'they'...</td>\n",
              "      <td>['O', 'O', 'O', 'O', 'O', 'O', 'O', 'O']</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>193085 rows × 2 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-8feab487-c3de-4881-9489-e3efe08168c1')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-8feab487-c3de-4881-9489-e3efe08168c1 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-8feab487-c3de-4881-9489-e3efe08168c1');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sb['ms_sentence'] = sb['ms_sentence'].str.replace('[',\"\")\n",
        "sb['ms_sentence'] = sb['ms_sentence'].str.replace(']',\"\")\n",
        "sb['ms_disfl'] = sb['ms_disfl'].str.replace('[',\"\")\n",
        "sb['ms_disfl'] = sb['ms_disfl'].str.replace(']',\"\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kDf1cjepQ0yR",
        "outputId": "5c130a28-272a-4724-da48-8559f09478d6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-8-53bb95292bf7>:1: FutureWarning: The default value of regex will change from True to False in a future version. In addition, single character regular expressions will *not* be treated as literal strings when regex=True.\n",
            "  sb['ms_sentence'] = sb['ms_sentence'].str.replace('[',\"\")\n",
            "<ipython-input-8-53bb95292bf7>:2: FutureWarning: The default value of regex will change from True to False in a future version. In addition, single character regular expressions will *not* be treated as literal strings when regex=True.\n",
            "  sb['ms_sentence'] = sb['ms_sentence'].str.replace(']',\"\")\n",
            "<ipython-input-8-53bb95292bf7>:3: FutureWarning: The default value of regex will change from True to False in a future version. In addition, single character regular expressions will *not* be treated as literal strings when regex=True.\n",
            "  sb['ms_disfl'] = sb['ms_disfl'].str.replace('[',\"\")\n",
            "<ipython-input-8-53bb95292bf7>:4: FutureWarning: The default value of regex will change from True to False in a future version. In addition, single character regular expressions will *not* be treated as literal strings when regex=True.\n",
            "  sb['ms_disfl'] = sb['ms_disfl'].str.replace(']',\"\")\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sb.dropna(axis = 0,inplace = True)"
      ],
      "metadata": {
        "id": "5ltcu0nPQ2tB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sb"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "hFEJ8KJ8Q5L_",
        "outputId": "53528f75-a159-428f-eebc-ae842d6a181b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                              ms_sentence  \\\n",
              "0                                                  'okay'   \n",
              "1       'uh', 'first', 'um', 'i', 'need', 'to', 'know'...   \n",
              "2       'well', 'of', 'course', 'it', \"'s\", 'you', 'kn...   \n",
              "3                                                   'yes'   \n",
              "4                                                  'yeah'   \n",
              "...                                                   ...   \n",
              "193080  'cause', 'i', 'have', \"n't\", 'been', 'there', ...   \n",
              "193081  'well', 'you', 'know', 'y-', 'going', 'over', ...   \n",
              "193082                                             'yeah'   \n",
              "193083  'you', 'can', 'see', 'that', 'horrible', 'horr...   \n",
              "193084  'well', 'we', 'were', 'there', 'when', 'they',...   \n",
              "\n",
              "                                                 ms_disfl  \n",
              "0                                                     'O'  \n",
              "1       'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'BE_IP...  \n",
              "2       'O', 'O', 'O', 'BE', 'IP', 'C', 'C', 'C', 'C',...  \n",
              "3                                                     'O'  \n",
              "4                                                     'O'  \n",
              "...                                                   ...  \n",
              "193080                  'O', 'O', 'O', 'O', 'O', 'O', 'O'  \n",
              "193081  'O', 'O', 'O', 'BE_IP', 'O', 'O', 'O', 'O', 'O...  \n",
              "193082                                                'O'  \n",
              "193083             'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O'  \n",
              "193084             'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O'  \n",
              "\n",
              "[193085 rows x 2 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-38ab01a1-dac4-447e-a4c1-a6bb5303e814\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>ms_sentence</th>\n",
              "      <th>ms_disfl</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>'okay'</td>\n",
              "      <td>'O'</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>'uh', 'first', 'um', 'i', 'need', 'to', 'know'...</td>\n",
              "      <td>'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'BE_IP...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>'well', 'of', 'course', 'it', \"'s\", 'you', 'kn...</td>\n",
              "      <td>'O', 'O', 'O', 'BE', 'IP', 'C', 'C', 'C', 'C',...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>'yes'</td>\n",
              "      <td>'O'</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>'yeah'</td>\n",
              "      <td>'O'</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>193080</th>\n",
              "      <td>'cause', 'i', 'have', \"n't\", 'been', 'there', ...</td>\n",
              "      <td>'O', 'O', 'O', 'O', 'O', 'O', 'O'</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>193081</th>\n",
              "      <td>'well', 'you', 'know', 'y-', 'going', 'over', ...</td>\n",
              "      <td>'O', 'O', 'O', 'BE_IP', 'O', 'O', 'O', 'O', 'O...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>193082</th>\n",
              "      <td>'yeah'</td>\n",
              "      <td>'O'</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>193083</th>\n",
              "      <td>'you', 'can', 'see', 'that', 'horrible', 'horr...</td>\n",
              "      <td>'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O'</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>193084</th>\n",
              "      <td>'well', 'we', 'were', 'there', 'when', 'they',...</td>\n",
              "      <td>'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O'</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>193085 rows × 2 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-38ab01a1-dac4-447e-a4c1-a6bb5303e814')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-38ab01a1-dac4-447e-a4c1-a6bb5303e814 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-38ab01a1-dac4-447e-a4c1-a6bb5303e814');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sb['ms_sentence'] = sb['ms_sentence'].str.replace(\"'\",\"\")\n",
        "sb['Sentences'] = sb['ms_sentence'].str.split(', ')\n",
        "sb['ms_disfl'] = sb['ms_disfl'].str.replace(\"'\",\"\")\n",
        "sb['Labels'] = sb['ms_disfl'].str.split(', ')\n"
      ],
      "metadata": {
        "id": "km8AdSjUQ7E-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sb"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 502
        },
        "id": "NtTiIv3JQ8-A",
        "outputId": "b7694afb-c0bf-4610-a426-9775a6a48ca4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                              ms_sentence  \\\n",
              "0                                                    okay   \n",
              "1       uh, first, um, i, need, to, know, uh, h-, how,...   \n",
              "2       well, of, course, it, \"s\", you, know, it, \"s\",...   \n",
              "3                                                     yes   \n",
              "4                                                    yeah   \n",
              "...                                                   ...   \n",
              "193080        cause, i, have, \"nt\", been, there, probably   \n",
              "193081  well, you, know, y-, going, over, the, hills, ...   \n",
              "193082                                               yeah   \n",
              "193083  you, can, see, that, horrible, horrible, brown...   \n",
              "193084          well, we, were, there, when, they, had, a   \n",
              "\n",
              "                                                 ms_disfl  \\\n",
              "0                                                       O   \n",
              "1       O, O, O, O, O, O, O, O, BE_IP, C, O, O, O, BE_...   \n",
              "2       O, O, O, BE, IP, C, C, C, C, O, O, O, O, O, O,...   \n",
              "3                                                       O   \n",
              "4                                                       O   \n",
              "...                                                   ...   \n",
              "193080                                O, O, O, O, O, O, O   \n",
              "193081       O, O, O, BE_IP, O, O, O, O, O, O, O, O, O, O   \n",
              "193082                                                  O   \n",
              "193083                             O, O, O, O, O, O, O, O   \n",
              "193084                             O, O, O, O, O, O, O, O   \n",
              "\n",
              "                                                Sentences  \\\n",
              "0                                                  [okay]   \n",
              "1       [uh, first, um, i, need, to, know, uh, h-, how...   \n",
              "2       [well, of, course, it, \"s\", you, know, it, \"s\"...   \n",
              "3                                                   [yes]   \n",
              "4                                                  [yeah]   \n",
              "...                                                   ...   \n",
              "193080      [cause, i, have, \"nt\", been, there, probably]   \n",
              "193081  [well, you, know, y-, going, over, the, hills,...   \n",
              "193082                                             [yeah]   \n",
              "193083  [you, can, see, that, horrible, horrible, brow...   \n",
              "193084        [well, we, were, there, when, they, had, a]   \n",
              "\n",
              "                                                   Labels  \n",
              "0                                                     [O]  \n",
              "1       [O, O, O, O, O, O, O, O, BE_IP, C, O, O, O, BE...  \n",
              "2       [O, O, O, BE, IP, C, C, C, C, O, O, O, O, O, O...  \n",
              "3                                                     [O]  \n",
              "4                                                     [O]  \n",
              "...                                                   ...  \n",
              "193080                              [O, O, O, O, O, O, O]  \n",
              "193081     [O, O, O, BE_IP, O, O, O, O, O, O, O, O, O, O]  \n",
              "193082                                                [O]  \n",
              "193083                           [O, O, O, O, O, O, O, O]  \n",
              "193084                           [O, O, O, O, O, O, O, O]  \n",
              "\n",
              "[193085 rows x 4 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-707b1b87-0a93-4dd6-8b1d-727a4fe2fae9\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>ms_sentence</th>\n",
              "      <th>ms_disfl</th>\n",
              "      <th>Sentences</th>\n",
              "      <th>Labels</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>okay</td>\n",
              "      <td>O</td>\n",
              "      <td>[okay]</td>\n",
              "      <td>[O]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>uh, first, um, i, need, to, know, uh, h-, how,...</td>\n",
              "      <td>O, O, O, O, O, O, O, O, BE_IP, C, O, O, O, BE_...</td>\n",
              "      <td>[uh, first, um, i, need, to, know, uh, h-, how...</td>\n",
              "      <td>[O, O, O, O, O, O, O, O, BE_IP, C, O, O, O, BE...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>well, of, course, it, \"s\", you, know, it, \"s\",...</td>\n",
              "      <td>O, O, O, BE, IP, C, C, C, C, O, O, O, O, O, O,...</td>\n",
              "      <td>[well, of, course, it, \"s\", you, know, it, \"s\"...</td>\n",
              "      <td>[O, O, O, BE, IP, C, C, C, C, O, O, O, O, O, O...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>yes</td>\n",
              "      <td>O</td>\n",
              "      <td>[yes]</td>\n",
              "      <td>[O]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>yeah</td>\n",
              "      <td>O</td>\n",
              "      <td>[yeah]</td>\n",
              "      <td>[O]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>193080</th>\n",
              "      <td>cause, i, have, \"nt\", been, there, probably</td>\n",
              "      <td>O, O, O, O, O, O, O</td>\n",
              "      <td>[cause, i, have, \"nt\", been, there, probably]</td>\n",
              "      <td>[O, O, O, O, O, O, O]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>193081</th>\n",
              "      <td>well, you, know, y-, going, over, the, hills, ...</td>\n",
              "      <td>O, O, O, BE_IP, O, O, O, O, O, O, O, O, O, O</td>\n",
              "      <td>[well, you, know, y-, going, over, the, hills,...</td>\n",
              "      <td>[O, O, O, BE_IP, O, O, O, O, O, O, O, O, O, O]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>193082</th>\n",
              "      <td>yeah</td>\n",
              "      <td>O</td>\n",
              "      <td>[yeah]</td>\n",
              "      <td>[O]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>193083</th>\n",
              "      <td>you, can, see, that, horrible, horrible, brown...</td>\n",
              "      <td>O, O, O, O, O, O, O, O</td>\n",
              "      <td>[you, can, see, that, horrible, horrible, brow...</td>\n",
              "      <td>[O, O, O, O, O, O, O, O]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>193084</th>\n",
              "      <td>well, we, were, there, when, they, had, a</td>\n",
              "      <td>O, O, O, O, O, O, O, O</td>\n",
              "      <td>[well, we, were, there, when, they, had, a]</td>\n",
              "      <td>[O, O, O, O, O, O, O, O]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>193085 rows × 4 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-707b1b87-0a93-4dd6-8b1d-727a4fe2fae9')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-707b1b87-0a93-4dd6-8b1d-727a4fe2fae9 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-707b1b87-0a93-4dd6-8b1d-727a4fe2fae9');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "type(sb.iloc[0,2][0])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "O4yDTUJ0Q_Wu",
        "outputId": "adc3c74a-2a8a-40d8-f82d-269744865440"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "str"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sb.iloc[1,2]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "l54jYMmnRCIS",
        "outputId": "2d000f0b-915e-4c96-993c-2dbb3356f3a9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['uh',\n",
              " 'first',\n",
              " 'um',\n",
              " 'i',\n",
              " 'need',\n",
              " 'to',\n",
              " 'know',\n",
              " 'uh',\n",
              " 'h-',\n",
              " 'how',\n",
              " 'do',\n",
              " 'you',\n",
              " 'feel',\n",
              " 'about',\n",
              " 'uh',\n",
              " 'about',\n",
              " 'sending',\n",
              " 'um',\n",
              " 'an',\n",
              " 'elderly',\n",
              " 'uh',\n",
              " 'family',\n",
              " 'member',\n",
              " 'to',\n",
              " 'a',\n",
              " 'nursing',\n",
              " 'home']"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Define a function to replace \"O\" with 0 in each list\n",
        "def replace_o(lst):\n",
        "  return [1 if x != \"O\" else 0 for x in lst]\n",
        "\n",
        "# Apply the function to each value in the \"column_name\" column\n",
        "sb['Labels'] = sb['Labels'].apply(replace_o)\n"
      ],
      "metadata": {
        "id": "0H2duccERDyt"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sb['Label_len'] = sb['Labels'].apply(lambda x: len(x))\n",
        "sb['Sentence_len'] = sb['Sentences'].apply(lambda x: len(x))"
      ],
      "metadata": {
        "id": "ibS5dmPaRFyA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Sentences in which the labels are not equal to the words\n",
        "sb = sb[sb['Label_len']==sb['Sentence_len']]"
      ],
      "metadata": {
        "id": "zjkQnDpSRHhA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sb['Sentences'].iloc[1]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "U6aww0brRJOw",
        "outputId": "9cbc3006-bb69-405d-9fc5-145dd9da1370"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['uh',\n",
              " 'first',\n",
              " 'um',\n",
              " 'i',\n",
              " 'need',\n",
              " 'to',\n",
              " 'know',\n",
              " 'uh',\n",
              " 'h-',\n",
              " 'how',\n",
              " 'do',\n",
              " 'you',\n",
              " 'feel',\n",
              " 'about',\n",
              " 'uh',\n",
              " 'about',\n",
              " 'sending',\n",
              " 'um',\n",
              " 'an',\n",
              " 'elderly',\n",
              " 'uh',\n",
              " 'family',\n",
              " 'member',\n",
              " 'to',\n",
              " 'a',\n",
              " 'nursing',\n",
              " 'home']"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "disf = ['um','uh','uh-hum','uh-huh']\n",
        "def label_change(row):\n",
        "    for i in range(0,row['Label_len']):\n",
        "        if row['Sentences'][i] in disf:\n",
        "            row[\"Labels\"][i] = 1\n",
        "        else:\n",
        "            pass\n",
        "    return row['Labels']\n",
        "sb['Labels'] = sb.apply(label_change, axis = 1)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hJZ5pTf5RKjL",
        "outputId": "f2bc3517-ae18-4d85-8ce6-6cb91bd3a29e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-19-e143be30523e>:9: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  sb['Labels'] = sb.apply(label_change, axis = 1)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sb = sb[sb['Label_len']<40]"
      ],
      "metadata": {
        "id": "aAciaBHbRPDB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data = sb[['Sentences','Labels']]"
      ],
      "metadata": {
        "id": "4P29Cx-eRSFT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# install datasets\n",
        "!pip install datasets"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "K3iikitlDorO",
        "outputId": "4309199e-0d47-4d2b-f9fb-876c94f6a113"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting datasets\n",
            "  Downloading datasets-2.8.0-py3-none-any.whl (452 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m452.9/452.9 KB\u001b[0m \u001b[31m9.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: packaging in /usr/local/lib/python3.8/dist-packages (from datasets) (21.3)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.8/dist-packages (from datasets) (6.0)\n",
            "Requirement already satisfied: dill<0.3.7 in /usr/local/lib/python3.8/dist-packages (from datasets) (0.3.6)\n",
            "Requirement already satisfied: fsspec[http]>=2021.11.1 in /usr/local/lib/python3.8/dist-packages (from datasets) (2022.11.0)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.8/dist-packages (from datasets) (1.3.5)\n",
            "Collecting multiprocess\n",
            "  Downloading multiprocess-0.70.14-py38-none-any.whl (132 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m132.0/132.0 KB\u001b[0m \u001b[31m16.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pyarrow>=6.0.0 in /usr/local/lib/python3.8/dist-packages (from datasets) (9.0.0)\n",
            "Collecting huggingface-hub<1.0.0,>=0.2.0\n",
            "  Downloading huggingface_hub-0.11.1-py3-none-any.whl (182 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m182.4/182.4 KB\u001b[0m \u001b[31m23.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.8/dist-packages (from datasets) (1.21.6)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.8/dist-packages (from datasets) (3.8.3)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.8/dist-packages (from datasets) (2.25.1)\n",
            "Collecting xxhash\n",
            "  Downloading xxhash-3.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (213 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m213.0/213.0 KB\u001b[0m \u001b[31m22.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting responses<0.19\n",
            "  Downloading responses-0.18.0-py3-none-any.whl (38 kB)\n",
            "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.8/dist-packages (from datasets) (4.64.1)\n",
            "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (2.1.1)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (4.0.2)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (22.2.0)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (1.8.2)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (1.3.3)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (1.3.1)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (6.0.3)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.8/dist-packages (from huggingface-hub<1.0.0,>=0.2.0->datasets) (4.4.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.8/dist-packages (from huggingface-hub<1.0.0,>=0.2.0->datasets) (3.8.2)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.8/dist-packages (from packaging->datasets) (3.0.9)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.8/dist-packages (from requests>=2.19.0->datasets) (1.24.3)\n",
            "Requirement already satisfied: chardet<5,>=3.0.2 in /usr/local/lib/python3.8/dist-packages (from requests>=2.19.0->datasets) (4.0.0)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.8/dist-packages (from requests>=2.19.0->datasets) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.8/dist-packages (from requests>=2.19.0->datasets) (2022.12.7)\n",
            "Collecting urllib3<1.27,>=1.21.1\n",
            "  Downloading urllib3-1.26.13-py2.py3-none-any.whl (140 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m140.6/140.6 KB\u001b[0m \u001b[31m19.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.8/dist-packages (from pandas->datasets) (2022.7)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.8/dist-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.8/dist-packages (from python-dateutil>=2.7.3->pandas->datasets) (1.15.0)\n",
            "Installing collected packages: xxhash, urllib3, multiprocess, responses, huggingface-hub, datasets\n",
            "  Attempting uninstall: urllib3\n",
            "    Found existing installation: urllib3 1.24.3\n",
            "    Uninstalling urllib3-1.24.3:\n",
            "      Successfully uninstalled urllib3-1.24.3\n",
            "Successfully installed datasets-2.8.0 huggingface-hub-0.11.1 multiprocess-0.70.14 responses-0.18.0 urllib3-1.26.13 xxhash-3.2.0\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "urllib3"
                ]
              }
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data = data.reset_index(drop = False)\n",
        "data"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "AhnXkFvoFr4z",
        "outputId": "37136004-601c-4209-bb62-e08f97f6e0be"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "         index                                          Sentences  \\\n",
              "0            0                                             [okay]   \n",
              "1            1  [uh, first, um, i, need, to, know, uh, h-, how...   \n",
              "2            3                                              [yes]   \n",
              "3            4                                             [yeah]   \n",
              "4            5  [i, \"d\", be, very, very, careful, and, uh, you...   \n",
              "...        ...                                                ...   \n",
              "173774  193080      [cause, i, have, \"nt\", been, there, probably]   \n",
              "173775  193081  [well, you, know, y-, going, over, the, hills,...   \n",
              "173776  193082                                             [yeah]   \n",
              "173777  193083  [you, can, see, that, horrible, horrible, brow...   \n",
              "173778  193084        [well, we, were, there, when, they, had, a]   \n",
              "\n",
              "                                                   Labels  \n",
              "0                                                     [0]  \n",
              "1       [1, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 1, ...  \n",
              "2                                                     [0]  \n",
              "3                                                     [0]  \n",
              "4                 [0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0]  \n",
              "...                                                   ...  \n",
              "173774                              [0, 0, 0, 0, 0, 0, 0]  \n",
              "173775         [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]  \n",
              "173776                                                [0]  \n",
              "173777                           [0, 0, 0, 0, 0, 0, 0, 0]  \n",
              "173778                           [0, 0, 0, 0, 0, 0, 0, 0]  \n",
              "\n",
              "[173779 rows x 3 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-7b187a81-7e54-4ff5-aeb4-67df8095ca94\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>index</th>\n",
              "      <th>Sentences</th>\n",
              "      <th>Labels</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>[okay]</td>\n",
              "      <td>[0]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>[uh, first, um, i, need, to, know, uh, h-, how...</td>\n",
              "      <td>[1, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 1, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3</td>\n",
              "      <td>[yes]</td>\n",
              "      <td>[0]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4</td>\n",
              "      <td>[yeah]</td>\n",
              "      <td>[0]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>5</td>\n",
              "      <td>[i, \"d\", be, very, very, careful, and, uh, you...</td>\n",
              "      <td>[0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>173774</th>\n",
              "      <td>193080</td>\n",
              "      <td>[cause, i, have, \"nt\", been, there, probably]</td>\n",
              "      <td>[0, 0, 0, 0, 0, 0, 0]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>173775</th>\n",
              "      <td>193081</td>\n",
              "      <td>[well, you, know, y-, going, over, the, hills,...</td>\n",
              "      <td>[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>173776</th>\n",
              "      <td>193082</td>\n",
              "      <td>[yeah]</td>\n",
              "      <td>[0]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>173777</th>\n",
              "      <td>193083</td>\n",
              "      <td>[you, can, see, that, horrible, horrible, brow...</td>\n",
              "      <td>[0, 0, 0, 0, 0, 0, 0, 0]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>173778</th>\n",
              "      <td>193084</td>\n",
              "      <td>[well, we, were, there, when, they, had, a]</td>\n",
              "      <td>[0, 0, 0, 0, 0, 0, 0, 0]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>173779 rows × 3 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-7b187a81-7e54-4ff5-aeb4-67df8095ca94')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-7b187a81-7e54-4ff5-aeb4-67df8095ca94 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-7b187a81-7e54-4ff5-aeb4-67df8095ca94');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data.rename(columns = {'index':'id','Sentences':'tokens','Labels':'ner_tags'},inplace = True)\n",
        "data"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "Te7OzK-UGEob",
        "outputId": "45db1c4a-7539-4bf8-92b6-b96eb0d236a2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "            id                                             tokens  \\\n",
              "0            0                                             [okay]   \n",
              "1            1  [uh, first, um, i, need, to, know, uh, h-, how...   \n",
              "2            3                                              [yes]   \n",
              "3            4                                             [yeah]   \n",
              "4            5  [i, \"d\", be, very, very, careful, and, uh, you...   \n",
              "...        ...                                                ...   \n",
              "173774  193080      [cause, i, have, \"nt\", been, there, probably]   \n",
              "173775  193081  [well, you, know, y-, going, over, the, hills,...   \n",
              "173776  193082                                             [yeah]   \n",
              "173777  193083  [you, can, see, that, horrible, horrible, brow...   \n",
              "173778  193084        [well, we, were, there, when, they, had, a]   \n",
              "\n",
              "                                                 ner_tags  \n",
              "0                                                     [0]  \n",
              "1       [1, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 1, ...  \n",
              "2                                                     [0]  \n",
              "3                                                     [0]  \n",
              "4                 [0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0]  \n",
              "...                                                   ...  \n",
              "173774                              [0, 0, 0, 0, 0, 0, 0]  \n",
              "173775         [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]  \n",
              "173776                                                [0]  \n",
              "173777                           [0, 0, 0, 0, 0, 0, 0, 0]  \n",
              "173778                           [0, 0, 0, 0, 0, 0, 0, 0]  \n",
              "\n",
              "[173779 rows x 3 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-1672b7c6-c6e5-46e5-b676-adffdc736a1d\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>tokens</th>\n",
              "      <th>ner_tags</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>[okay]</td>\n",
              "      <td>[0]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>[uh, first, um, i, need, to, know, uh, h-, how...</td>\n",
              "      <td>[1, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 1, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3</td>\n",
              "      <td>[yes]</td>\n",
              "      <td>[0]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4</td>\n",
              "      <td>[yeah]</td>\n",
              "      <td>[0]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>5</td>\n",
              "      <td>[i, \"d\", be, very, very, careful, and, uh, you...</td>\n",
              "      <td>[0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>173774</th>\n",
              "      <td>193080</td>\n",
              "      <td>[cause, i, have, \"nt\", been, there, probably]</td>\n",
              "      <td>[0, 0, 0, 0, 0, 0, 0]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>173775</th>\n",
              "      <td>193081</td>\n",
              "      <td>[well, you, know, y-, going, over, the, hills,...</td>\n",
              "      <td>[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>173776</th>\n",
              "      <td>193082</td>\n",
              "      <td>[yeah]</td>\n",
              "      <td>[0]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>173777</th>\n",
              "      <td>193083</td>\n",
              "      <td>[you, can, see, that, horrible, horrible, brow...</td>\n",
              "      <td>[0, 0, 0, 0, 0, 0, 0, 0]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>173778</th>\n",
              "      <td>193084</td>\n",
              "      <td>[well, we, were, there, when, they, had, a]</td>\n",
              "      <td>[0, 0, 0, 0, 0, 0, 0, 0]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>173779 rows × 3 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-1672b7c6-c6e5-46e5-b676-adffdc736a1d')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-1672b7c6-c6e5-46e5-b676-adffdc736a1d button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-1672b7c6-c6e5-46e5-b676-adffdc736a1d');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Using Huggingface data loader & data loader"
      ],
      "metadata": {
        "id": "doDQoq0RHqZA"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from datasets import Dataset, DatasetDict\n",
        "data = Dataset.from_pandas(data,preserve_index = False)"
      ],
      "metadata": {
        "id": "rnLdnhSTDLHU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "dataset = data.train_test_split()"
      ],
      "metadata": {
        "id": "ZdLVKrwBDuV0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(type(dataset),len(dataset))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "po6OZrRPJfAC",
        "outputId": "872b40f3-4c9c-42a9-860e-9b19a5de2d8d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<class 'datasets.dataset_dict.DatasetDict'> 2\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "dataset"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mw6zzmu_Jodc",
        "outputId": "9f86bcaf-c643-41b6-8bde-4fec6902a2ba"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DatasetDict({\n",
              "    train: Dataset({\n",
              "        features: ['id', 'tokens', 'ner_tags'],\n",
              "        num_rows: 130334\n",
              "    })\n",
              "    test: Dataset({\n",
              "        features: ['id', 'tokens', 'ner_tags'],\n",
              "        num_rows: 43445\n",
              "    })\n",
              "})"
            ]
          },
          "metadata": {},
          "execution_count": 69
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "dataset['train'][0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kbL02aw1J6SE",
        "outputId": "26773ae9-7e06-498a-b59d-c57c4bc3232b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'id': 12936,\n",
              " 'tokens': ['and',\n",
              "  'uh',\n",
              "  'i',\n",
              "  'can',\n",
              "  '\"nt\"',\n",
              "  'remember',\n",
              "  'me',\n",
              "  'and',\n",
              "  'i',\n",
              "  'think',\n",
              "  'me',\n",
              "  'and',\n",
              "  'dana',\n",
              "  'had',\n",
              "  'football'],\n",
              " 'ner_tags': [0, 1, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0, 0, 0]}"
            ]
          },
          "metadata": {},
          "execution_count": 70
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install transformers"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SHjGHjXxIOuV",
        "outputId": "ddf2a234-7971-45c6-c90d-ea2cc5a2bb51"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting transformers\n",
            "  Downloading transformers-4.25.1-py3-none-any.whl (5.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.8/5.8 MB\u001b[0m \u001b[31m49.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.8/dist-packages (from transformers) (21.3)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.8/dist-packages (from transformers) (2.25.1)\n",
            "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1\n",
            "  Downloading tokenizers-0.13.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.6/7.6 MB\u001b[0m \u001b[31m103.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.8/dist-packages (from transformers) (4.64.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.8/dist-packages (from transformers) (3.8.2)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.8/dist-packages (from transformers) (1.21.6)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.8/dist-packages (from transformers) (6.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.8/dist-packages (from transformers) (2022.6.2)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.10.0 in /usr/local/lib/python3.8/dist-packages (from transformers) (0.11.1)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.8/dist-packages (from huggingface-hub<1.0,>=0.10.0->transformers) (4.4.0)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.8/dist-packages (from packaging>=20.0->transformers) (3.0.9)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.8/dist-packages (from requests->transformers) (1.26.13)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.8/dist-packages (from requests->transformers) (2022.12.7)\n",
            "Requirement already satisfied: chardet<5,>=3.0.2 in /usr/local/lib/python3.8/dist-packages (from requests->transformers) (4.0.0)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.8/dist-packages (from requests->transformers) (2.10)\n",
            "Installing collected packages: tokenizers, transformers\n",
            "Successfully installed tokenizers-0.13.2 transformers-4.25.1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import AutoTokenizer\n",
        "\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"distilbert-base-uncased\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 145,
          "referenced_widgets": [
            "4e9fdbbecf144aca84af38ae052a9c21",
            "2ddefd0514c84487a6e8014266e09ab1",
            "7033ac4552b24f3db3566396a3ada30e",
            "20393b312c3b4f80bd9974cdd6de6a57",
            "3e6a7c2a7b754494aa42decc922f8cac",
            "3f66ecafc4794a4396f488333b66931d",
            "2832d33b924540868c2fea13e9a24f4e",
            "a9063ea34fae4237a2defbb809f832fd",
            "986babfb154d459f883343a8a125ba90",
            "31e3b629cdf644d89f648c895a558a95",
            "6129c4ed8944451ab40cfb5db52e592a",
            "8fb5dbddce094de098a13713dd508ff0",
            "7a29931518d2495da197329e3b3a29a3",
            "bbaa16bf555341e89b7eb12eee42b4c0",
            "e7447f907b6c43288c6aa783b5de6d7d",
            "214b6b59bb0b4ed1a871cb4c531c3da7",
            "ea43d255f38c47dd834ebfec72b36fa6",
            "3eeef0fa9d71455d837b7e561600591e",
            "fe37108810cd46cb9bad2ad869641e4e",
            "d7eb8f19c6e14fe8bc688bac2c5a1646",
            "cfc4f9daa1c64c10b7b658eeb1f90ed0",
            "f9898223aeea4665856e162d0ae6c59f",
            "398774bdb22f4160acd0c34537e6f2a5",
            "908fc2d0fc8c4c4b8213bbb320dba612",
            "8992f3272951416aa4d9d6b0d8d6308d",
            "1a6ba27915624faf816133c0a82ea800",
            "e2c23a48c58f4660ad04316a172343e1",
            "9ec4a273114d4fff8a2909a80210b2a7",
            "893c89b6b2aa4a2eb998c4ca40c763ca",
            "78fc985159134280bf9d5c587a3beb8d",
            "b35d13bdeb4a4139ac592a5e45c9d173",
            "e1d3742130c24d129ca0600bf246a63f",
            "9a33c0a344f04398a084b2887deadb36",
            "7dbc46b2f79f44da86c94029aec234d2",
            "3e0554557d3d43d58d4307d93851a330",
            "e326bee990a048788b30ed6e53268abb",
            "6bf8533fe35b4ba9b7ac99f5b943acae",
            "55f69c54fff445e4a35e362638717235",
            "8a6224cb72f84cde8b0e43d503852b42",
            "93ea24773200455288bb18b2c7d8957e",
            "55f703c630cd4206a1637e43ebe928ca",
            "6659dd86bf364c1485ad9180b5f7986c",
            "a2626bc8e35643789807f60ec1def0b1",
            "996de5d6ce744144b8270ece86785c1c"
          ]
        },
        "id": "E7SZ1ohKIAhj",
        "outputId": "6eea3e8a-8bfb-4edf-a981-8fc76b96fd08"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/28.0 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "4e9fdbbecf144aca84af38ae052a9c21"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/483 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "8fb5dbddce094de098a13713dd508ff0"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/232k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "398774bdb22f4160acd0c34537e6f2a5"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/466k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "7dbc46b2f79f44da86c94029aec234d2"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "example = dataset['train'][6]\n",
        "tokenized_input = tokenizer(example[\"tokens\"], is_split_into_words=True)\n",
        "tokens = tokenizer.convert_ids_to_tokens(tokenized_input[\"input_ids\"])\n",
        "tokens"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rLLhSj7aIUJG",
        "outputId": "35f5771b-64ea-4e54-f1e1-52489e16db6c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['[CLS]', 'yeah', 'uh', '[SEP]']"
            ]
          },
          "metadata": {},
          "execution_count": 76
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "dataset[1]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xc_eVtalIZhs",
        "outputId": "0c6c2bf3-2e5f-463a-99c2-bb2f571b0c15"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'id': 1,\n",
              " 'tokens': ['uh',\n",
              "  'first',\n",
              "  'um',\n",
              "  'i',\n",
              "  'need',\n",
              "  'to',\n",
              "  'know',\n",
              "  'uh',\n",
              "  'h-',\n",
              "  'how',\n",
              "  'do',\n",
              "  'you',\n",
              "  'feel',\n",
              "  'about',\n",
              "  'uh',\n",
              "  'about',\n",
              "  'sending',\n",
              "  'um',\n",
              "  'an',\n",
              "  'elderly',\n",
              "  'uh',\n",
              "  'family',\n",
              "  'member',\n",
              "  'to',\n",
              "  'a',\n",
              "  'nursing',\n",
              "  'home'],\n",
              " 'ner_tags': [1,\n",
              "  0,\n",
              "  1,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  1,\n",
              "  1,\n",
              "  1,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  1,\n",
              "  1,\n",
              "  1,\n",
              "  0,\n",
              "  1,\n",
              "  0,\n",
              "  0,\n",
              "  1,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0]}"
            ]
          },
          "metadata": {},
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def tokenize_and_align_labels(examples):\n",
        "    tokenized_inputs = tokenizer(examples[\"tokens\"], truncation=True, is_split_into_words=True)\n",
        "\n",
        "    labels = []\n",
        "    for i, label in enumerate(examples[f\"ner_tags\"]):\n",
        "        word_ids = tokenized_inputs.word_ids(batch_index=i)  # Map tokens to their respective word.\n",
        "        previous_word_idx = None\n",
        "        label_ids = []\n",
        "        for word_idx in word_ids:  # Set the special tokens to -100.\n",
        "            if word_idx is None:\n",
        "                label_ids.append(-100)\n",
        "            elif word_idx != previous_word_idx:  # Only label the first token of a given word.\n",
        "                label_ids.append(label[word_idx])\n",
        "            else:\n",
        "                label_ids.append(-100)\n",
        "            previous_word_idx = word_idx\n",
        "        labels.append(label_ids)\n",
        "\n",
        "    tokenized_inputs[\"labels\"] = labels\n",
        "    return tokenized_inputs"
      ],
      "metadata": {
        "id": "aoVnKjqQIEht"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tokenized_data = dataset.map(tokenize_and_align_labels, batched=True)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 81,
          "referenced_widgets": [
            "482ac6fa4d9a492ba6a5523e57a31d42",
            "a264a29dfa6d40f09b00b170f851fc0d",
            "e32f8c7c25e7412493ba27ddcfe8eb70",
            "918c082ee0f64447bdc8b9b0eeaa1dfb",
            "101c8dd4de434f0384cbb365e474292f",
            "97f3c6d599e34b83a6a80983016309bf",
            "2fca42aa96114f0f8957ac9d710c62cb",
            "b615c3f3496e4a499ac1b78cfaffe227",
            "c06cf37e3ed448ecaff47c2cc43bfac9",
            "7ddd3f3194574ac5a4c5ceb2ae030a79",
            "74b433090de042bbbd183228aa2e3cc5",
            "8e1a5990c3e943a889cef35334602b9d",
            "60aa9d535bd34c12bc7149f2aaf0f5f8",
            "affa06d53b0c44d9b150936662feb51d",
            "0058dbdd1e694211a7a75154d7691604",
            "08b1e9027e0d4998abcfb2fe18bb6258",
            "d45b00dc5e864d5a8292a8818c3def5f",
            "de64b62c69ef44718e3322bc856ceab9",
            "eb7a54a43ad74bd99957b6fd84e07757",
            "c55ec19133e648be8418ab54c36b52e4",
            "4b1bdd96978f4789b01b8e52b52f20e5",
            "331fdc0e7a084ee983d334604623a05e"
          ]
        },
        "id": "E7hB_mHttBoD",
        "outputId": "eebf90bd-843c-4b17-85de-65d9b625301b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/131 [00:00<?, ?ba/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "482ac6fa4d9a492ba6a5523e57a31d42"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/44 [00:00<?, ?ba/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "8e1a5990c3e943a889cef35334602b9d"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "type(tokenized_data)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IHwvalxJIuSJ",
        "outputId": "c1200f9c-a319-400b-dd1c-d9276a3a9457"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "datasets.dataset_dict.DatasetDict"
            ]
          },
          "metadata": {},
          "execution_count": 79
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tokenized_data"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "H8gdRpEopelr",
        "outputId": "48ea7359-a772-4604-b02c-c0e80c8c6acf"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DatasetDict({\n",
              "    train: Dataset({\n",
              "        features: ['id', 'tokens', 'ner_tags', 'input_ids', 'attention_mask', 'labels'],\n",
              "        num_rows: 130334\n",
              "    })\n",
              "    test: Dataset({\n",
              "        features: ['id', 'tokens', 'ner_tags', 'input_ids', 'attention_mask', 'labels'],\n",
              "        num_rows: 43445\n",
              "    })\n",
              "})"
            ]
          },
          "metadata": {},
          "execution_count": 80
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import DataCollatorForTokenClassification\n",
        "\n",
        "data_collator = DataCollatorForTokenClassification(tokenizer=tokenizer, return_tensors=\"tf\")"
      ],
      "metadata": {
        "id": "1B3L-8c4LfGl"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install evaluate\n",
        "!pip install seqeval"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "f7O6QshqLmbv",
        "outputId": "4ee88217-6230-429c-c8eb-4c0c3435fa74"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: evaluate in /usr/local/lib/python3.8/dist-packages (0.4.0)\n",
            "Requirement already satisfied: responses<0.19 in /usr/local/lib/python3.8/dist-packages (from evaluate) (0.18.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.8/dist-packages (from evaluate) (21.3)\n",
            "Requirement already satisfied: fsspec[http]>=2021.05.0 in /usr/local/lib/python3.8/dist-packages (from evaluate) (2022.11.0)\n",
            "Requirement already satisfied: xxhash in /usr/local/lib/python3.8/dist-packages (from evaluate) (3.2.0)\n",
            "Requirement already satisfied: dill in /usr/local/lib/python3.8/dist-packages (from evaluate) (0.3.6)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.8/dist-packages (from evaluate) (1.3.5)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.8/dist-packages (from evaluate) (2.25.1)\n",
            "Requirement already satisfied: datasets>=2.0.0 in /usr/local/lib/python3.8/dist-packages (from evaluate) (2.8.0)\n",
            "Requirement already satisfied: huggingface-hub>=0.7.0 in /usr/local/lib/python3.8/dist-packages (from evaluate) (0.11.1)\n",
            "Requirement already satisfied: multiprocess in /usr/local/lib/python3.8/dist-packages (from evaluate) (0.70.14)\n",
            "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.8/dist-packages (from evaluate) (4.64.1)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.8/dist-packages (from evaluate) (1.21.6)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.8/dist-packages (from datasets>=2.0.0->evaluate) (6.0)\n",
            "Requirement already satisfied: pyarrow>=6.0.0 in /usr/local/lib/python3.8/dist-packages (from datasets>=2.0.0->evaluate) (9.0.0)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.8/dist-packages (from datasets>=2.0.0->evaluate) (3.8.3)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.8/dist-packages (from huggingface-hub>=0.7.0->evaluate) (4.4.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.8/dist-packages (from huggingface-hub>=0.7.0->evaluate) (3.8.2)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.8/dist-packages (from packaging->evaluate) (3.0.9)\n",
            "Requirement already satisfied: chardet<5,>=3.0.2 in /usr/local/lib/python3.8/dist-packages (from requests>=2.19.0->evaluate) (4.0.0)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.8/dist-packages (from requests>=2.19.0->evaluate) (1.26.13)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.8/dist-packages (from requests>=2.19.0->evaluate) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.8/dist-packages (from requests>=2.19.0->evaluate) (2022.12.7)\n",
            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.8/dist-packages (from pandas->evaluate) (2022.7)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.8/dist-packages (from pandas->evaluate) (2.8.2)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.8.2)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.3.1)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets>=2.0.0->evaluate) (6.0.3)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.3.3)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets>=2.0.0->evaluate) (22.2.0)\n",
            "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets>=2.0.0->evaluate) (2.1.1)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets>=2.0.0->evaluate) (4.0.2)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.8/dist-packages (from python-dateutil>=2.7.3->pandas->evaluate) (1.15.0)\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: seqeval in /usr/local/lib/python3.8/dist-packages (1.2.2)\n",
            "Requirement already satisfied: numpy>=1.14.0 in /usr/local/lib/python3.8/dist-packages (from seqeval) (1.21.6)\n",
            "Requirement already satisfied: scikit-learn>=0.21.3 in /usr/local/lib/python3.8/dist-packages (from seqeval) (1.0.2)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.8/dist-packages (from scikit-learn>=0.21.3->seqeval) (3.1.0)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.8/dist-packages (from scikit-learn>=0.21.3->seqeval) (1.2.0)\n",
            "Requirement already satisfied: scipy>=1.1.0 in /usr/local/lib/python3.8/dist-packages (from scikit-learn>=0.21.3->seqeval) (1.7.3)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import evaluate\n",
        "\n",
        "seqeval = evaluate.load(\"seqeval\")"
      ],
      "metadata": {
        "id": "4W4UKASzLhUV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Compute metrics function"
      ],
      "metadata": {
        "id": "_EkuDDszrIZZ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "label_list = ['Fluent','Disfluent']\n",
        "\n",
        "\n",
        "def compute_metrics(p):\n",
        "    predictions, labels = p\n",
        "    predictions = np.argmax(predictions, axis=2)\n",
        "\n",
        "    true_predictions = [\n",
        "        [label_list[p] for (p, l) in zip(prediction, label) if l != -100]\n",
        "        for prediction, label in zip(predictions, labels)\n",
        "    ]\n",
        "    true_labels = [\n",
        "        [label_list[l] for (p, l) in zip(prediction, label) if l != -100]\n",
        "        for prediction, label in zip(predictions, labels)\n",
        "    ]\n",
        "\n",
        "    results = seqeval.compute(predictions=true_predictions, references=true_labels)\n",
        "    return {\n",
        "        \"precision\": results[\"overall_precision\"],\n",
        "        \"recall\": results[\"overall_recall\"],\n",
        "        \"f1\": results[\"overall_f1\"],\n",
        "        \"accuracy\": results[\"overall_accuracy\"],\n",
        "    }"
      ],
      "metadata": {
        "id": "rR3sESo9oo7U"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "id2label = {\n",
        "    0: \"Fluent\",\n",
        "    1: \"Disfluent\"\n",
        "}\n",
        "label2id = {\n",
        "    \"Fluent\": 0,\n",
        "    \"Disfluent\": 1\n",
        "}"
      ],
      "metadata": {
        "id": "SlusEs5mo_Ww"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Optimizer"
      ],
      "metadata": {
        "id": "-86nLPVhrMCP"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import create_optimizer\n",
        "\n",
        "batch_size = 16\n",
        "num_train_epochs = 3\n",
        "num_train_steps = (len(tokenized_data['train']) // batch_size) * num_train_epochs\n",
        "optimizer, lr_schedule = create_optimizer(\n",
        "    init_lr=2e-5,\n",
        "    num_train_steps=num_train_steps,\n",
        "    weight_decay_rate=0.01,\n",
        "    num_warmup_steps=0,\n",
        ")"
      ],
      "metadata": {
        "id": "Tkf83KoxpUPM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import TFAutoModelForTokenClassification\n",
        "\n",
        "model = TFAutoModelForTokenClassification.from_pretrained(\n",
        "    \"distilbert-base-uncased\", num_labels=2, id2label=id2label, label2id=label2id\n",
        ")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2rqSwIAtsdJG",
        "outputId": "6a7744cf-4504-4f2e-f531-e25f583b9f77"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some layers from the model checkpoint at distilbert-base-uncased were not used when initializing TFDistilBertForTokenClassification: ['activation_13', 'vocab_layer_norm', 'vocab_transform', 'vocab_projector']\n",
            "- This IS expected if you are initializing TFDistilBertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing TFDistilBertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some layers of TFDistilBertForTokenClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['dropout_59', 'classifier']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tf_train_set = model.prepare_tf_dataset(\n",
        "    tokenized_data['train'],\n",
        "    shuffle=True,\n",
        "    batch_size=16,\n",
        "    collate_fn=data_collator,\n",
        ")\n",
        "\n",
        "tf_validation_set = model.prepare_tf_dataset(\n",
        "    tokenized_data['test'],\n",
        "    shuffle=False,\n",
        "    batch_size=16,\n",
        "    collate_fn=data_collator,\n",
        ")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Y2jMQt_qsemB",
        "outputId": "03da7b59-7126-4e6f-8aed-89c7e11b4ff4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.8/dist-packages/transformers/tokenization_utils_base.py:717: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray.\n",
            "  tensor = as_tensor(value)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model.compile(optimizer=optimizer)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hcWntYkXLLCS",
        "outputId": "3238e7c1-1d0b-49bb-ed75-57029c354e33"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "No loss specified in compile() - the model's internal loss computation will be used as the loss. Don't panic - this is a common way to train TensorFlow models in Transformers! To disable this behaviour please pass a loss argument, or explicitly pass `loss=None` if you do not want your model to compute a loss.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers.keras_callbacks import KerasMetricCallback\n",
        "\n",
        "metric_callback = KerasMetricCallback(metric_fn=compute_metrics, eval_dataset=tf_validation_set)"
      ],
      "metadata": {
        "id": "3FU7bpZYLW_Q"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from huggingface_hub import notebook_login\n",
        "notebook_login()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 332,
          "referenced_widgets": [
            "e6cdb0252bd7423da90d66b8e792e81e",
            "08c37914eba7475a9970d19d9ccf2775",
            "90019d1d25f14c378ade0511a9ebaf25",
            "db41f900658344ee83907a6312920554",
            "4920a5b7c981417a986c5f89fd51740c",
            "fda380adb4e24d0da76ab3e2470272b1",
            "53923d483cea46cab5475e7fe1b7412b",
            "44cf7477ab1e4c7597d3d3f9390acbff",
            "731ee20423bf43d691488dec048ea8d8",
            "f33de199c5fd435395a4689bc04f06fe",
            "90e3ff69547d4409a9bdfb983aa8be51",
            "3cffbadafafd4628909ca58d839818d2",
            "41c2a366d94b47359d6f951f037bd043",
            "2fe2e876b8d341ffa7a7a4d86826c8c2",
            "72cf14297fe04d3499b1a6b2cbdb2327",
            "39405232588841ffb7e4dc7de1cd861c",
            "e1212d03d77d42659e91f2923f6248e7"
          ]
        },
        "id": "9gjqjzb3Q0xZ",
        "outputId": "9501a09f-21cd-4285-8263-e79f47b88254"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "e6cdb0252bd7423da90d66b8e792e81e"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers.keras_callbacks import PushToHubCallback\n",
        "push_to_hub_callback = PushToHubCallback('model',\n",
        "    hub_model_id=\"Aditya02/Speech_Analyzer_Model\",\n",
        "    tokenizer=tokenizer\n",
        ")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lPfZ-MwvLZ68",
        "outputId": "cf7d0d79-9078-49e6-ae72-dd8dac2c47ee"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/content/model is already a clone of https://huggingface.co/Aditya02/Speech_Analyzer_Model. Make sure you pull the latest changes with `repo.git_pull()`.\n",
            "WARNING:huggingface_hub.repository:/content/model is already a clone of https://huggingface.co/Aditya02/Speech_Analyzer_Model. Make sure you pull the latest changes with `repo.git_pull()`.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "callbacks = [metric_callback, push_to_hub_callback]"
      ],
      "metadata": {
        "id": "SArGPXALL0QY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model.fit(x=tf_train_set, validation_data=tf_validation_set, epochs=3, callbacks=callbacks)"
      ],
      "metadata": {
        "id": "97BVRHVPL3gf"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "text = \"uh I am here trying out this model for uh for disfluency. I um think it will do good.\""
      ],
      "metadata": {
        "id": "NknYld7yMKFq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import pipeline\n",
        "\n",
        "classifier = pipeline(\"ner\", model=\"/content/model\")\n",
        "classifier(text)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9ldD0_TYc83l",
        "outputId": "56b2a221-ac84-4a8a-d9c0-57150c7cee24"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some layers from the model checkpoint at /content/model were not used when initializing TFDistilBertForTokenClassification: ['dropout_59']\n",
            "- This IS expected if you are initializing TFDistilBertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing TFDistilBertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some layers of TFDistilBertForTokenClassification were not initialized from the model checkpoint at /content/model and are newly initialized: ['dropout_79']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'entity': 'Disfluent',\n",
              "  'score': 0.9997907,\n",
              "  'index': 1,\n",
              "  'word': 'uh',\n",
              "  'start': 0,\n",
              "  'end': 2},\n",
              " {'entity': 'Fluent',\n",
              "  'score': 0.99894184,\n",
              "  'index': 2,\n",
              "  'word': 'i',\n",
              "  'start': 3,\n",
              "  'end': 4},\n",
              " {'entity': 'Fluent',\n",
              "  'score': 0.9986608,\n",
              "  'index': 3,\n",
              "  'word': 'am',\n",
              "  'start': 5,\n",
              "  'end': 7},\n",
              " {'entity': 'Fluent',\n",
              "  'score': 0.99859923,\n",
              "  'index': 4,\n",
              "  'word': 'here',\n",
              "  'start': 8,\n",
              "  'end': 12},\n",
              " {'entity': 'Fluent',\n",
              "  'score': 0.9986953,\n",
              "  'index': 5,\n",
              "  'word': 'trying',\n",
              "  'start': 13,\n",
              "  'end': 19},\n",
              " {'entity': 'Fluent',\n",
              "  'score': 0.9988134,\n",
              "  'index': 6,\n",
              "  'word': 'out',\n",
              "  'start': 20,\n",
              "  'end': 23},\n",
              " {'entity': 'Fluent',\n",
              "  'score': 0.9987472,\n",
              "  'index': 7,\n",
              "  'word': 'this',\n",
              "  'start': 24,\n",
              "  'end': 28},\n",
              " {'entity': 'Fluent',\n",
              "  'score': 0.999091,\n",
              "  'index': 8,\n",
              "  'word': 'model',\n",
              "  'start': 29,\n",
              "  'end': 34},\n",
              " {'entity': 'Disfluent',\n",
              "  'score': 0.98563045,\n",
              "  'index': 9,\n",
              "  'word': 'for',\n",
              "  'start': 35,\n",
              "  'end': 38},\n",
              " {'entity': 'Disfluent',\n",
              "  'score': 0.9987558,\n",
              "  'index': 10,\n",
              "  'word': 'uh',\n",
              "  'start': 39,\n",
              "  'end': 41},\n",
              " {'entity': 'Disfluent',\n",
              "  'score': 0.9806011,\n",
              "  'index': 11,\n",
              "  'word': 'for',\n",
              "  'start': 42,\n",
              "  'end': 45},\n",
              " {'entity': 'Fluent',\n",
              "  'score': 0.9963792,\n",
              "  'index': 12,\n",
              "  'word': 'di',\n",
              "  'start': 46,\n",
              "  'end': 48},\n",
              " {'entity': 'Fluent',\n",
              "  'score': 0.99863607,\n",
              "  'index': 13,\n",
              "  'word': '##sf',\n",
              "  'start': 48,\n",
              "  'end': 50},\n",
              " {'entity': 'Fluent',\n",
              "  'score': 0.98066807,\n",
              "  'index': 14,\n",
              "  'word': '##lu',\n",
              "  'start': 50,\n",
              "  'end': 52},\n",
              " {'entity': 'Fluent',\n",
              "  'score': 0.9990514,\n",
              "  'index': 15,\n",
              "  'word': '##ency',\n",
              "  'start': 52,\n",
              "  'end': 56},\n",
              " {'entity': 'Fluent',\n",
              "  'score': 0.95205945,\n",
              "  'index': 16,\n",
              "  'word': '.',\n",
              "  'start': 56,\n",
              "  'end': 57},\n",
              " {'entity': 'Fluent',\n",
              "  'score': 0.9959312,\n",
              "  'index': 17,\n",
              "  'word': 'i',\n",
              "  'start': 58,\n",
              "  'end': 59},\n",
              " {'entity': 'Disfluent',\n",
              "  'score': 0.999603,\n",
              "  'index': 18,\n",
              "  'word': 'um',\n",
              "  'start': 60,\n",
              "  'end': 62},\n",
              " {'entity': 'Fluent',\n",
              "  'score': 0.9979377,\n",
              "  'index': 19,\n",
              "  'word': 'think',\n",
              "  'start': 63,\n",
              "  'end': 68},\n",
              " {'entity': 'Fluent',\n",
              "  'score': 0.99940395,\n",
              "  'index': 20,\n",
              "  'word': 'it',\n",
              "  'start': 69,\n",
              "  'end': 71},\n",
              " {'entity': 'Fluent',\n",
              "  'score': 0.99898666,\n",
              "  'index': 21,\n",
              "  'word': 'will',\n",
              "  'start': 72,\n",
              "  'end': 76},\n",
              " {'entity': 'Fluent',\n",
              "  'score': 0.9991523,\n",
              "  'index': 22,\n",
              "  'word': 'do',\n",
              "  'start': 77,\n",
              "  'end': 79},\n",
              " {'entity': 'Fluent',\n",
              "  'score': 0.9994591,\n",
              "  'index': 23,\n",
              "  'word': 'good',\n",
              "  'start': 80,\n",
              "  'end': 84},\n",
              " {'entity': 'Fluent',\n",
              "  'score': 0.9961378,\n",
              "  'index': 24,\n",
              "  'word': '.',\n",
              "  'start': 84,\n",
              "  'end': 85}]"
            ]
          },
          "metadata": {},
          "execution_count": 122
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!zip -r /content/model.zip /content/model"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Xnm-5EwptVJa",
        "outputId": "8f0e89e2-dcc7-412b-e7b4-f5885c4ee2fb"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "  adding: content/model/ (stored 0%)\n",
            "  adding: content/model/tokenizer.json (deflated 71%)\n",
            "  adding: content/model/vocab.txt (deflated 53%)\n",
            "  adding: content/model/.git/ (stored 0%)\n",
            "  adding: content/model/.git/COMMIT_EDITMSG (stored 0%)\n",
            "  adding: content/model/.git/logs/ (stored 0%)\n",
            "  adding: content/model/.git/logs/HEAD (deflated 36%)\n",
            "  adding: content/model/.git/logs/refs/ (stored 0%)\n",
            "  adding: content/model/.git/logs/refs/heads/ (stored 0%)\n",
            "  adding: content/model/.git/logs/refs/heads/main (deflated 36%)\n",
            "  adding: content/model/.git/logs/refs/remotes/ (stored 0%)\n",
            "  adding: content/model/.git/logs/refs/remotes/origin/ (stored 0%)\n",
            "  adding: content/model/.git/logs/refs/remotes/origin/HEAD (deflated 22%)\n",
            "  adding: content/model/.git/info/ (stored 0%)\n",
            "  adding: content/model/.git/info/exclude (deflated 28%)\n",
            "  adding: content/model/.git/packed-refs (deflated 10%)\n",
            "  adding: content/model/.git/description (deflated 14%)\n",
            "  adding: content/model/.git/index (deflated 33%)\n",
            "  adding: content/model/.git/HEAD (stored 0%)\n",
            "  adding: content/model/.git/branches/ (stored 0%)\n",
            "  adding: content/model/.git/hooks/ (stored 0%)\n",
            "  adding: content/model/.git/hooks/prepare-commit-msg.sample (deflated 50%)\n",
            "  adding: content/model/.git/hooks/post-merge (deflated 25%)\n",
            "  adding: content/model/.git/hooks/pre-push.sample (deflated 50%)\n",
            "  adding: content/model/.git/hooks/commit-msg.sample (deflated 44%)\n",
            "  adding: content/model/.git/hooks/fsmonitor-watchman.sample (deflated 53%)\n",
            "  adding: content/model/.git/hooks/pre-push (deflated 24%)\n",
            "  adding: content/model/.git/hooks/post-commit (deflated 25%)\n",
            "  adding: content/model/.git/hooks/pre-rebase.sample (deflated 59%)\n",
            "  adding: content/model/.git/hooks/update.sample (deflated 68%)\n",
            "  adding: content/model/.git/hooks/post-checkout (deflated 26%)\n",
            "  adding: content/model/.git/hooks/applypatch-msg.sample (deflated 42%)\n",
            "  adding: content/model/.git/hooks/post-update.sample (deflated 27%)\n",
            "  adding: content/model/.git/hooks/pre-applypatch.sample (deflated 38%)\n",
            "  adding: content/model/.git/hooks/pre-receive.sample (deflated 40%)\n",
            "  adding: content/model/.git/hooks/pre-commit.sample (deflated 43%)\n",
            "  adding: content/model/.git/objects/ (stored 0%)\n",
            "  adding: content/model/.git/objects/info/ (stored 0%)\n",
            "  adding: content/model/.git/objects/a8/ (stored 0%)\n",
            "  adding: content/model/.git/objects/a8/b3208c2884c4efb86e49300fdd3dc877220cdf (stored 0%)\n",
            "  adding: content/model/.git/objects/35/ (stored 0%)\n",
            "  adding: content/model/.git/objects/35/7265304d4444848a1e4802d38d8a15fee7393d (stored 0%)\n",
            "  adding: content/model/.git/objects/00/ (stored 0%)\n",
            "  adding: content/model/.git/objects/00/4b2c4cdcd844239008803e81e4df6f4c0539ce (stored 0%)\n",
            "  adding: content/model/.git/objects/57/ (stored 0%)\n",
            "  adding: content/model/.git/objects/57/6f3b9df3079a849bac1d5a6c5c57d2951c0fd5 (stored 0%)\n",
            "  adding: content/model/.git/objects/e7/ (stored 0%)\n",
            "  adding: content/model/.git/objects/e7/d90ddb789e264c5ea4a99dccd0b911883df4c5 (stored 0%)\n",
            "  adding: content/model/.git/objects/39/ (stored 0%)\n",
            "  adding: content/model/.git/objects/39/97cf8f77c508399b28956da47dd158fe314c84 (stored 0%)\n",
            "  adding: content/model/.git/objects/e2/ (stored 0%)\n",
            "  adding: content/model/.git/objects/e2/a1a7d64feb4c2127be90a75ab60c87acfec777 (stored 0%)\n",
            "  adding: content/model/.git/objects/05/ (stored 0%)\n",
            "  adding: content/model/.git/objects/05/a08d51432c8ca05501d170fbc36142a93a163e (deflated 1%)\n",
            "  adding: content/model/.git/objects/c7/ (stored 0%)\n",
            "  adding: content/model/.git/objects/c7/d9f3332a950355d5a77d85000f05e6f45435ea (stored 0%)\n",
            "  adding: content/model/.git/objects/pack/ (stored 0%)\n",
            "  adding: content/model/.git/objects/7b/ (stored 0%)\n",
            "  adding: content/model/.git/objects/7b/7468f389f5ce555996c08da4c52209d7c890a7 (stored 0%)\n",
            "  adding: content/model/.git/objects/fb/ (stored 0%)\n",
            "  adding: content/model/.git/objects/fb/140275c155a9c7c5a3b3e0e77a9e839594a938 (deflated 2%)\n",
            "  adding: content/model/.git/objects/20/ (stored 0%)\n",
            "  adding: content/model/.git/objects/20/75450c1a119fb4226cd626c91f72c831af9af6 (stored 0%)\n",
            "  adding: content/model/.git/objects/cf/ (stored 0%)\n",
            "  adding: content/model/.git/objects/cf/c80b6cea547d5390498c2046423c70bd035279 (stored 0%)\n",
            "  adding: content/model/.git/refs/ (stored 0%)\n",
            "  adding: content/model/.git/refs/heads/ (stored 0%)\n",
            "  adding: content/model/.git/refs/heads/main (stored 0%)\n",
            "  adding: content/model/.git/refs/remotes/ (stored 0%)\n",
            "  adding: content/model/.git/refs/remotes/origin/ (stored 0%)\n",
            "  adding: content/model/.git/refs/remotes/origin/HEAD (stored 0%)\n",
            "  adding: content/model/.git/refs/tags/ (stored 0%)\n",
            "  adding: content/model/.git/lfs/ (stored 0%)\n",
            "  adding: content/model/.git/lfs/objects/ (stored 0%)\n",
            "  adding: content/model/.git/lfs/objects/logs/ (stored 0%)\n",
            "  adding: content/model/.git/lfs/objects/6a/ (stored 0%)\n",
            "  adding: content/model/.git/lfs/objects/6a/09/ (stored 0%)\n",
            "  adding: content/model/.git/lfs/objects/6a/09/6a09bb75bf0cfcdb834fc16894da043db325386128002b6d759900f00d9e05a9 (deflated 8%)\n",
            "  adding: content/model/.git/lfs/tmp/ (stored 0%)\n",
            "  adding: content/model/.git/lfs/tmp/objects/ (stored 0%)\n",
            "  adding: content/model/.git/config (deflated 45%)\n",
            "  adding: content/model/README.md (deflated 53%)\n",
            "  adding: content/model/special_tokens_map.json (deflated 42%)\n",
            "  adding: content/model/config.json (deflated 46%)\n",
            "  adding: content/model/tf_model.h5 (deflated 8%)\n",
            "  adding: content/model/tokenizer_config.json (deflated 41%)\n",
            "  adding: content/model/.gitattributes (deflated 87%)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import files\n",
        "files.download('/content/model.zip')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "id": "Hk2qh9w7sf1X",
        "outputId": "77db4452-98cb-485f-b760-1cc658e57068"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "\n",
              "    async function download(id, filename, size) {\n",
              "      if (!google.colab.kernel.accessAllowed) {\n",
              "        return;\n",
              "      }\n",
              "      const div = document.createElement('div');\n",
              "      const label = document.createElement('label');\n",
              "      label.textContent = `Downloading \"${filename}\": `;\n",
              "      div.appendChild(label);\n",
              "      const progress = document.createElement('progress');\n",
              "      progress.max = size;\n",
              "      div.appendChild(progress);\n",
              "      document.body.appendChild(div);\n",
              "\n",
              "      const buffers = [];\n",
              "      let downloaded = 0;\n",
              "\n",
              "      const channel = await google.colab.kernel.comms.open(id);\n",
              "      // Send a message to notify the kernel that we're ready.\n",
              "      channel.send({})\n",
              "\n",
              "      for await (const message of channel.messages) {\n",
              "        // Send a message to notify the kernel that we're ready.\n",
              "        channel.send({})\n",
              "        if (message.buffers) {\n",
              "          for (const buffer of message.buffers) {\n",
              "            buffers.push(buffer);\n",
              "            downloaded += buffer.byteLength;\n",
              "            progress.value = downloaded;\n",
              "          }\n",
              "        }\n",
              "      }\n",
              "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
              "      const a = document.createElement('a');\n",
              "      a.href = window.URL.createObjectURL(blob);\n",
              "      a.download = filename;\n",
              "      div.appendChild(a);\n",
              "      a.click();\n",
              "      div.remove();\n",
              "    }\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "download(\"download_d0d45f99-30b1-4fcd-9707-865527ebf696\", \"model.zip\", 489922951)"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "lQUKCU2PtECd"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}