distilBERT_colab.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "Copia_di_distilBERT.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "toxic",
      "language": "python",
      "name": "toxic"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.10"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "33a664acd9f24f4483f25ce00ad6973c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_b782542d157640bca560c94d0e443c0f",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_b6eb97c0eec54ae8932db4bdac623c1c",
              "IPY_MODEL_fa20b09bac124647b418bc9c9a3ca838",
              "IPY_MODEL_f5efaba9a5174abc8b5b8da91a82176a"
            ]
          }
        },
        "b782542d157640bca560c94d0e443c0f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b6eb97c0eec54ae8932db4bdac623c1c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_58e628544b054249833d2076659466fe",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8abb329e976d485ba974fac386e4c2b3"
          }
        },
        "fa20b09bac124647b418bc9c9a3ca838": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_89f288d56189434db12b2634fa3cfc23",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 231508,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 231508,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_70d56eb0f310436794f6a123545ee1de"
          }
        },
        "f5efaba9a5174abc8b5b8da91a82176a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_3c6c619e3538404eb0450ee05229248c",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 232k/232k [00:00&lt;00:00, 892kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_76aaff8c528f45f6b08943255be30e3a"
          }
        },
        "58e628544b054249833d2076659466fe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8abb329e976d485ba974fac386e4c2b3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "89f288d56189434db12b2634fa3cfc23": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "70d56eb0f310436794f6a123545ee1de": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3c6c619e3538404eb0450ee05229248c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "76aaff8c528f45f6b08943255be30e3a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "bb9a0bfd44a647928e340a82307403e5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_ef86051afaab49dfa8e168fec96e8fe3",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_e491a5790a6740d68f25559ba8ec7049",
              "IPY_MODEL_c61e4c7a35724831ae098fdaa34a0b5e",
              "IPY_MODEL_83507f2d686e4da8a54d08c8fc60fb1b"
            ]
          }
        },
        "ef86051afaab49dfa8e168fec96e8fe3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e491a5790a6740d68f25559ba8ec7049": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_1b87d51a03624a8e9711d70d4d41acd4",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f6d621966407400dbdb4ec201c622c31"
          }
        },
        "c61e4c7a35724831ae098fdaa34a0b5e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_9d679bdaeba249c8b325612047103b20",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 466062,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 466062,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d0bde0dc80f04e81b449f7ad5bce1e2f"
          }
        },
        "83507f2d686e4da8a54d08c8fc60fb1b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_f97bfdc811d34da99be999f8ba77a116",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 466k/466k [00:00&lt;00:00, 1.24MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3d2a569a619244fab7a65096c30fb29c"
          }
        },
        "1b87d51a03624a8e9711d70d4d41acd4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f6d621966407400dbdb4ec201c622c31": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9d679bdaeba249c8b325612047103b20": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d0bde0dc80f04e81b449f7ad5bce1e2f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f97bfdc811d34da99be999f8ba77a116": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3d2a569a619244fab7a65096c30fb29c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6eac4ac6962c4bd3874431e461bafdd6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_5c99f586f8754019854e1994fb542363",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_abb1002ba2974e21b6a3773f11315f6b",
              "IPY_MODEL_7fdcbd7faa714e4cb04f16590a2e687b",
              "IPY_MODEL_7cc8d0d7c1d14219a55c832372dfd1cb"
            ]
          }
        },
        "5c99f586f8754019854e1994fb542363": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "abb1002ba2974e21b6a3773f11315f6b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_ba44897a0331462f9bd0cd295a4b01a5",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_75a1411465284d1fa7f4dabc3e0bdc9e"
          }
        },
        "7fdcbd7faa714e4cb04f16590a2e687b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_4e972170e80b445185df777253f63531",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 28,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 28,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d9fdfdc3b65143b8a6fb79682b3cdb4b"
          }
        },
        "7cc8d0d7c1d14219a55c832372dfd1cb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_640a183e9d854a18a58ec9f22f1164be",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 28.0/28.0 [00:00&lt;00:00, 568B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7788d400fd204ed5b957ce4a775e53c7"
          }
        },
        "ba44897a0331462f9bd0cd295a4b01a5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "75a1411465284d1fa7f4dabc3e0bdc9e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4e972170e80b445185df777253f63531": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d9fdfdc3b65143b8a6fb79682b3cdb4b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "640a183e9d854a18a58ec9f22f1164be": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7788d400fd204ed5b957ce4a775e53c7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "92f4b3b411944c509b0f02bf7d295cea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_886d5dfb75ca45d98d36778925a4170c",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_bb51ff0351f64401b782e7cd1753859b",
              "IPY_MODEL_0d8e038a6044458184fdcb183f81acbc",
              "IPY_MODEL_a78676d8f486446298fb3c500fd56bcf"
            ]
          }
        },
        "886d5dfb75ca45d98d36778925a4170c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "bb51ff0351f64401b782e7cd1753859b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_32095daed67d4360a9ae12ad29eb10b4",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7e20fa01dd6144e38cc61c5c1512fba2"
          }
        },
        "0d8e038a6044458184fdcb183f81acbc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_cd06040d435b4aea8bac70a0744d7897",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 483,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 483,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f11e1a4ea7d2499c98cd0c01eaeae69a"
          }
        },
        "a78676d8f486446298fb3c500fd56bcf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_98ece3d755f2495b9435aa385f60c524",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 483/483 [00:00&lt;00:00, 11.3kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0878eadec8da4e7e8447e101e568e791"
          }
        },
        "32095daed67d4360a9ae12ad29eb10b4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7e20fa01dd6144e38cc61c5c1512fba2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "cd06040d435b4aea8bac70a0744d7897": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f11e1a4ea7d2499c98cd0c01eaeae69a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "98ece3d755f2495b9435aa385f60c524": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0878eadec8da4e7e8447e101e568e791": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4885ed9ece684e63a8e23937fa5437d6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_bb05725ba42e44db9d04d0cebf7b16d5",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_85fa358ebea34e8eba44a5164663f113",
              "IPY_MODEL_dc79f458e1db47af8e59369291c745e2",
              "IPY_MODEL_eea6c294ab604230b363753a230e98d3"
            ]
          }
        },
        "bb05725ba42e44db9d04d0cebf7b16d5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "85fa358ebea34e8eba44a5164663f113": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_8c3d45e8ae5b42feb9491761b5d5a137",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7471c42b64694212a34ce3bebf6030ff"
          }
        },
        "dc79f458e1db47af8e59369291c745e2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_911ebfb9f140401bb0069ac17e832010",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 363423424,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 363423424,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f175583938394d49b9b01e31049ac565"
          }
        },
        "eea6c294ab604230b363753a230e98d3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_12e31a4d681f437990fe3271f3e6187b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 363M/363M [00:11&lt;00:00, 32.0MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_536245b0d3b245ecb5a68efcc97a1d56"
          }
        },
        "8c3d45e8ae5b42feb9491761b5d5a137": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7471c42b64694212a34ce3bebf6030ff": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "911ebfb9f140401bb0069ac17e832010": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f175583938394d49b9b01e31049ac565": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "12e31a4d681f437990fe3271f3e6187b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "536245b0d3b245ecb5a68efcc97a1d56": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xPQUdNYEXNG3"
      },
      "source": [
        "# distilBERT <br/>\n",
        "https://towardsdatascience.com/hugging-face-transformers-fine-tuning-distilbert-for-binary-classification-tasks-490f1d192379"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hpzf5E-tvyW7",
        "outputId": "c9076fd6-3ca7-4e95-ce25-60cc16ceb66b"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive') # /content/drive/MyDrive/HLT"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "i8AvDjNqXokn",
        "outputId": "6352880e-160a-46ba-8e4b-e980a681a517"
      },
      "source": [
        "!pip install transformers"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting transformers\n",
            "  Downloading transformers-4.10.2-py3-none-any.whl (2.8 MB)\n",
            "\u001b[K     |████████████████████████████████| 2.8 MB 5.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.8.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Collecting huggingface-hub>=0.0.12\n",
            "  Downloading huggingface_hub-0.0.17-py3-none-any.whl (52 kB)\n",
            "\u001b[K     |████████████████████████████████| 52 kB 1.5 MB/s \n",
            "\u001b[?25hCollecting tokenizers<0.11,>=0.10.1\n",
            "  Downloading tokenizers-0.10.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (3.3 MB)\n",
            "\u001b[K     |████████████████████████████████| 3.3 MB 38.5 MB/s \n",
            "\u001b[?25hCollecting pyyaml>=5.1\n",
            "  Downloading PyYAML-5.4.1-cp37-cp37m-manylinux1_x86_64.whl (636 kB)\n",
            "\u001b[K     |████████████████████████████████| 636 kB 37.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.19.5)\n",
            "Collecting sacremoses\n",
            "  Downloading sacremoses-0.0.45-py3-none-any.whl (895 kB)\n",
            "\u001b[K     |████████████████████████████████| 895 kB 37.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers) (21.0)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.62.2)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from huggingface-hub>=0.0.12->transformers) (3.7.4.3)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.5.0)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2021.5.30)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.0.1)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Installing collected packages: tokenizers, sacremoses, pyyaml, huggingface-hub, transformers\n",
            "  Attempting uninstall: pyyaml\n",
            "    Found existing installation: PyYAML 3.13\n",
            "    Uninstalling PyYAML-3.13:\n",
            "      Successfully uninstalled PyYAML-3.13\n",
            "Successfully installed huggingface-hub-0.0.17 pyyaml-5.4.1 sacremoses-0.0.45 tokenizers-0.10.3 transformers-4.10.2\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5MCpqtvGXO97",
        "outputId": "9bc1171a-f81b-4689-b6df-8cc5c9b8c9ed"
      },
      "source": [
        "!git clone https://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@github.com/antodima/toxic-comment.git\n",
        "!mv toxic-comment/data/ ."
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'toxic-comment'...\n",
            "remote: Enumerating objects: 76, done.\u001b[K\n",
            "remote: Counting objects: 100% (76/76), done.\u001b[K\n",
            "remote: Compressing objects: 100% (52/52), done.\u001b[K\n",
            "remote: Total 76 (delta 31), reused 66 (delta 21), pack-reused 0\u001b[K\n",
            "Unpacking objects: 100% (76/76), done.\n",
            "Checking out files: 100% (18/18), done.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dB2hdYnuXNG5"
      },
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import datetime\n",
        "from sklearn.metrics import accuracy_score, coverage_error\n",
        "import tensorflow as tf\n",
        "from tensorflow.keras.callbacks import ModelCheckpoint\n",
        "tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)\n",
        "from sklearn.model_selection import train_test_split\n",
        "from transformers import TFDistilBertModel, DistilBertTokenizerFast\n",
        "\n",
        "tf.compat.v1.enable_eager_execution()"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "X_YhB8dZXiqf",
        "outputId": "7e8ae3c8-67e9-4038-92a1-1a7de3d2180b"
      },
      "source": [
        "print(\"Num GPUs Available: \", len(tf.config.list_physical_devices('GPU')))"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Num GPUs Available:  1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h64jXVxUXNG6"
      },
      "source": [
        "def batch_encode(tokenizer, texts, batch_size=256, max_length=100):\n",
        "    \"\"\"\"\"\"\"\"\"\n",
        "    A function that encodes a batch of texts and returns the texts'\n",
        "    corresponding encodings and attention masks that are ready to be fed \n",
        "    into a pre-trained transformer model.\n",
        "    \n",
        "    Input:\n",
        "        - tokenizer:   Tokenizer object from the PreTrainedTokenizer Class\n",
        "        - texts:       List of strings where each string represents a text\n",
        "        - batch_size:  Integer controlling number of texts in a batch\n",
        "        - max_length:  Integer controlling max number of words to tokenize in a given text\n",
        "    Output:\n",
        "        - input_ids:       sequence of texts encoded as a tf.Tensor object\n",
        "        - attention_mask:  the texts' attention mask encoded as a tf.Tensor object\n",
        "    \"\"\"\"\"\"\"\"\"\n",
        "    \n",
        "    input_ids = []\n",
        "    attention_mask = []\n",
        "    \n",
        "    for i in range(0, len(texts), batch_size):\n",
        "        batch = texts[i:i+batch_size]\n",
        "        inputs = tokenizer.batch_encode_plus(batch, max_length=max_length, padding='longest', truncation=True, \n",
        "                                             return_attention_mask=True, return_token_type_ids=False)\n",
        "        input_ids.extend(inputs['input_ids'])\n",
        "        attention_mask.extend(inputs['attention_mask'])\n",
        "        \n",
        "    return tf.convert_to_tensor(input_ids), tf.convert_to_tensor(attention_mask)"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wTbcLSAXXNG7",
        "outputId": "5df05b89-933a-4441-e826-54a53ae6cd69"
      },
      "source": [
        "df = pd.read_csv('data/train.csv')\n",
        "categories = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']\n",
        "num_classes = len(categories)\n",
        "comments = list(df.comment_text.values)\n",
        "\n",
        "X_train, X_valid, y_train, y_valid = train_test_split(df['comment_text'].values, df[categories].values, \n",
        "                                                      train_size=0.8, shuffle=True, random_state=42)\n",
        "print(X_train.shape,y_train.shape)\n",
        "print(X_valid.shape,y_valid.shape)"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(127656,) (127656, 6)\n",
            "(31915,) (31915, 6)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "d0NDKJUKXNG7",
        "outputId": "ce1e82f2-8ef1-441f-c00d-eeeeddf19ef6"
      },
      "source": [
        "df_test = pd.read_csv('data/test.csv')\n",
        "cols = df_test.columns\n",
        "label_cols = list(cols[2:])\n",
        "test_labels_df = pd.read_csv('data/test_labels.csv')\n",
        "df_test = df_test.merge(test_labels_df, on='id', how='left')\n",
        "test_label_cols = list(df_test.columns[2:])\n",
        "df_test = df_test[~df_test[test_label_cols].eq(-1).any(axis=1)] # remove irrelevant rows/comments with -1 values\n",
        "\n",
        "X_test = df_test['comment_text'].values\n",
        "y_test = df_test[categories].values\n",
        "\n",
        "print(X_test.shape,y_test.shape)"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(63978,) (63978, 6)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 284,
          "referenced_widgets": [
            "33a664acd9f24f4483f25ce00ad6973c",
            "b782542d157640bca560c94d0e443c0f",
            "b6eb97c0eec54ae8932db4bdac623c1c",
            "fa20b09bac124647b418bc9c9a3ca838",
            "f5efaba9a5174abc8b5b8da91a82176a",
            "58e628544b054249833d2076659466fe",
            "8abb329e976d485ba974fac386e4c2b3",
            "89f288d56189434db12b2634fa3cfc23",
            "70d56eb0f310436794f6a123545ee1de",
            "3c6c619e3538404eb0450ee05229248c",
            "76aaff8c528f45f6b08943255be30e3a",
            "bb9a0bfd44a647928e340a82307403e5",
            "ef86051afaab49dfa8e168fec96e8fe3",
            "e491a5790a6740d68f25559ba8ec7049",
            "c61e4c7a35724831ae098fdaa34a0b5e",
            "83507f2d686e4da8a54d08c8fc60fb1b",
            "1b87d51a03624a8e9711d70d4d41acd4",
            "f6d621966407400dbdb4ec201c622c31",
            "9d679bdaeba249c8b325612047103b20",
            "d0bde0dc80f04e81b449f7ad5bce1e2f",
            "f97bfdc811d34da99be999f8ba77a116",
            "3d2a569a619244fab7a65096c30fb29c",
            "6eac4ac6962c4bd3874431e461bafdd6",
            "5c99f586f8754019854e1994fb542363",
            "abb1002ba2974e21b6a3773f11315f6b",
            "7fdcbd7faa714e4cb04f16590a2e687b",
            "7cc8d0d7c1d14219a55c832372dfd1cb",
            "ba44897a0331462f9bd0cd295a4b01a5",
            "75a1411465284d1fa7f4dabc3e0bdc9e",
            "4e972170e80b445185df777253f63531",
            "d9fdfdc3b65143b8a6fb79682b3cdb4b",
            "640a183e9d854a18a58ec9f22f1164be",
            "7788d400fd204ed5b957ce4a775e53c7",
            "92f4b3b411944c509b0f02bf7d295cea",
            "886d5dfb75ca45d98d36778925a4170c",
            "bb51ff0351f64401b782e7cd1753859b",
            "0d8e038a6044458184fdcb183f81acbc",
            "a78676d8f486446298fb3c500fd56bcf",
            "32095daed67d4360a9ae12ad29eb10b4",
            "7e20fa01dd6144e38cc61c5c1512fba2",
            "cd06040d435b4aea8bac70a0744d7897",
            "f11e1a4ea7d2499c98cd0c01eaeae69a",
            "98ece3d755f2495b9435aa385f60c524",
            "0878eadec8da4e7e8447e101e568e791",
            "4885ed9ece684e63a8e23937fa5437d6",
            "bb05725ba42e44db9d04d0cebf7b16d5",
            "85fa358ebea34e8eba44a5164663f113",
            "dc79f458e1db47af8e59369291c745e2",
            "eea6c294ab604230b363753a230e98d3",
            "8c3d45e8ae5b42feb9491761b5d5a137",
            "7471c42b64694212a34ce3bebf6030ff",
            "911ebfb9f140401bb0069ac17e832010",
            "f175583938394d49b9b01e31049ac565",
            "12e31a4d681f437990fe3271f3e6187b",
            "536245b0d3b245ecb5a68efcc97a1d56"
          ]
        },
        "id": "Esq345U3XNG8",
        "outputId": "423499fb-32d1-4730-8372-cce0ab1bea78"
      },
      "source": [
        "tokenizer = DistilBertTokenizerFast.from_pretrained('distilbert-base-uncased')\n",
        "distilBERT = TFDistilBertModel.from_pretrained('distilbert-base-uncased')\n",
        "for layer in distilBERT.layers:\n",
        "    layer.trainable = False"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "33a664acd9f24f4483f25ce00ad6973c",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/232k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "bb9a0bfd44a647928e340a82307403e5",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/466k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "6eac4ac6962c4bd3874431e461bafdd6",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/28.0 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "92f4b3b411944c509b0f02bf7d295cea",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/483 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "4885ed9ece684e63a8e23937fa5437d6",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/363M [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some layers from the model checkpoint at distilbert-base-uncased were not used when initializing TFDistilBertModel: ['activation_13', 'vocab_transform', 'vocab_layer_norm', 'vocab_projector']\n",
            "- This IS expected if you are initializing TFDistilBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing TFDistilBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "All the layers of TFDistilBertModel were initialized from the model checkpoint at distilbert-base-uncased.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFDistilBertModel for predictions without further training.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f2o5DL7qXNG8"
      },
      "source": [
        "max_length = 100\n",
        "# Encode X_train\n",
        "X_train_ids, X_train_attention = batch_encode(tokenizer, X_train.tolist(), max_length=max_length)\n",
        "# Encode X_valid\n",
        "X_valid_ids, X_valid_attention = batch_encode(tokenizer, X_valid.tolist(), max_length=max_length)\n",
        "# Encode X_test\n",
        "X_test_ids, X_test_attention = batch_encode(tokenizer, X_test.tolist(), max_length=max_length)"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RX7OL_2FXNG9",
        "outputId": "665930bc-5f35-4dcf-b6c0-2cd148da07c0"
      },
      "source": [
        "input_ids_layer = tf.keras.layers.Input(shape=(max_length,), name='input_ids', dtype='int32')\n",
        "input_attention_layer = tf.keras.layers.Input(shape=(max_length,), name='input_attention', dtype='int32')\n",
        "last_hidden_state = distilBERT([input_ids_layer, input_attention_layer])[0]\n",
        "cls_token = last_hidden_state[:, 0, :]\n",
        "output = tf.keras.layers.Dense(50, activation=\"relu\", kernel_initializer='he_uniform')(cls_token)\n",
        "output = tf.keras.layers.Dropout(0.1)(output)\n",
        "output = tf.keras.layers.Dense(num_classes, activation='sigmoid', name='classifier', kernel_initializer='glorot_uniform')(output)\n",
        "\n",
        "model = tf.keras.Model([input_ids_layer, input_attention_layer], output, name='')\n",
        "model.compile(optimizer='adam', \n",
        "              loss='binary_crossentropy', \n",
        "              metrics=[\n",
        "                  'accuracy',\n",
        "                  'binary_accuracy', \n",
        "                  tf.keras.metrics.AUC(multi_label=True),\n",
        "                  tf.keras.metrics.Precision(),\n",
        "                  tf.keras.metrics.Recall()\n",
        "              ])\n",
        "model.summary()"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_ids (InputLayer)          [(None, 100)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "input_attention (InputLayer)    [(None, 100)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "tf_distil_bert_model (TFDistilB TFBaseModelOutput(la 66362880    input_ids[0][0]                  \n",
            "                                                                 input_attention[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "tf.__operators__.getitem (Slici (None, 768)          0           tf_distil_bert_model[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "dense (Dense)                   (None, 50)           38450       tf.__operators__.getitem[0][0]   \n",
            "__________________________________________________________________________________________________\n",
            "dropout_19 (Dropout)            (None, 50)           0           dense[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "classifier (Dense)              (None, 6)            306         dropout_19[0][0]                 \n",
            "==================================================================================================\n",
            "Total params: 66,401,636\n",
            "Trainable params: 38,756\n",
            "Non-trainable params: 66,362,880\n",
            "__________________________________________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7RY6PuBZXNG9",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "03808708-07c7-41c6-ee0a-d6a9954cb0da"
      },
      "source": [
        "mode = 2  # 0: train, 1: evaluate, 2: finetuning\n",
        "\n",
        "if mode == 0:  # training\n",
        "  now = datetime.datetime.now()\n",
        "  ckp_dir = f\"/content/drive/MyDrive/HLT/checkpoint_{now.strftime('%Y%m%d_%H%M%S')}/distil-bert\"\n",
        "  print(f\"Training {ckp_dir} . . .\")\n",
        "  epochs = 4\n",
        "  batch_size = 64\n",
        "  num_steps = len(X_train) // batch_size\n",
        "\n",
        "  with tf.device('/GPU:0'):\n",
        "    history = model.fit(\n",
        "        x = [X_train_ids, X_train_attention],\n",
        "        y = y_train,\n",
        "        epochs = epochs,\n",
        "        batch_size = batch_size,\n",
        "        steps_per_epoch = num_steps,\n",
        "        validation_data = ([X_valid_ids, X_valid_attention], y_valid),\n",
        "        verbose=1,\n",
        "        callbacks=[\n",
        "          ModelCheckpoint(filepath=ckp_dir, save_weights_only=True, monitor='val_binary_accuracy', mode='max', save_best_only=True)\n",
        "        ]\n",
        "    )\n",
        "\n",
        "    scores = model.evaluate([X_test_ids, X_test_attention], y_test, verbose=2)\n",
        "    print(f\"Test {model.metrics_names}: {scores}\")\n",
        "    scores = model.evaluate([X_valid_ids, X_valid_attention], y_valid, verbose=2)\n",
        "    print(f\"Validation {model.metrics_names}: {scores}\")\n",
        "\n",
        "elif mode == 1:  # evaluate\n",
        "  path = '/content/drive/MyDrive/HLT/checkpoint_20210920_105809/'\n",
        "  print(f\"Evaluating {path} . . .\")\n",
        "  model.load_weights(tf.train.latest_checkpoint(path))\n",
        "\n",
        "  scores = model.evaluate([X_test_ids, X_test_attention], y_test, verbose=2)\n",
        "  print(f\"Test {model.metrics_names}: {scores}\")\n",
        "  scores = model.evaluate([X_valid_ids, X_valid_attention], y_valid, verbose=2)\n",
        "  print(f\"Validation {model.metrics_names}: {scores}\")\n",
        "\n",
        "elif mode == 2:  # fnetuning\n",
        "  path = '/content/drive/MyDrive/HLT/checkpoint_20210920_195726/'\n",
        "  model.load_weights(tf.train.latest_checkpoint(path))\n",
        "  print(f\"Finetuning {path} . . .\")\n",
        "  for layer in model.layers[-5].layers:\n",
        "    layer.trainable = True\n",
        "  model.summary()\n",
        "\n",
        "  epochs = 2\n",
        "  batch_size = 64\n",
        "  num_steps = len(X_train) // batch_size\n",
        "\n",
        "  now = datetime.datetime.now()\n",
        "  ckp_dir = f\"/content/drive/MyDrive/HLT/checkpoint_ft_{now.strftime('%Y%m%d_%H%M%S')}/distil-bert-finetuned\"\n",
        "\n",
        "  with tf.device('/GPU:0'):\n",
        "    history = model.fit(\n",
        "        x = [X_train_ids, X_train_attention],\n",
        "        y = y_train,\n",
        "        epochs = epochs,\n",
        "        batch_size = batch_size,\n",
        "        steps_per_epoch = num_steps,\n",
        "        validation_data = ([X_valid_ids, X_valid_attention], y_valid),\n",
        "        verbose=1,\n",
        "        callbacks=[\n",
        "          ModelCheckpoint(filepath=ckp_dir, save_weights_only=True, monitor='val_binary_accuracy', mode='max', save_best_only=True)\n",
        "        ]\n",
        "    )\n",
        "\n",
        "  scores = model.evaluate([X_test_ids, X_test_attention], y_test, verbose=2)\n",
        "  print(f\"Test {model.metrics_names}: {scores}\")\n",
        "  scores = model.evaluate([X_valid_ids, X_valid_attention], y_valid, verbose=2)\n",
        "  print(f\"Validation {model.metrics_names}: {scores}\")"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finetuning /content/drive/MyDrive/HLT/checkpoint_20210920_195726/ . . .\n",
            "Model: \"model\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_ids (InputLayer)          [(None, 100)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "input_attention (InputLayer)    [(None, 100)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "tf_distil_bert_model (TFDistilB TFBaseModelOutput(la 66362880    input_ids[0][0]                  \n",
            "                                                                 input_attention[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "tf.__operators__.getitem (Slici (None, 768)          0           tf_distil_bert_model[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "dense (Dense)                   (None, 50)           38450       tf.__operators__.getitem[0][0]   \n",
            "__________________________________________________________________________________________________\n",
            "dropout_19 (Dropout)            (None, 50)           0           dense[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "classifier (Dense)              (None, 6)            306         dropout_19[0][0]                 \n",
            "==================================================================================================\n",
            "Total params: 66,401,636\n",
            "Trainable params: 66,401,636\n",
            "Non-trainable params: 0\n",
            "__________________________________________________________________________________________________\n",
            "Epoch 1/2\n",
            "1994/1994 [==============================] - 1040s 518ms/step - loss: 0.0586 - accuracy: 0.9685 - binary_accuracy: 0.9787 - auc: 0.9618 - precision: 0.8050 - recall: 0.5506 - val_loss: 0.0592 - val_accuracy: 0.8595 - val_binary_accuracy: 0.9786 - val_auc: 0.9531 - val_precision: 0.8685 - val_recall: 0.4948\n",
            "Epoch 2/2\n",
            "1994/1994 [==============================] - 1027s 515ms/step - loss: 0.0583 - accuracy: 0.9689 - binary_accuracy: 0.9786 - auc: 0.9610 - precision: 0.8030 - recall: 0.5522 - val_loss: 0.0569 - val_accuracy: 0.9888 - val_binary_accuracy: 0.9791 - val_auc: 0.9544 - val_precision: 0.8337 - val_recall: 0.5413\n",
            "2000/2000 - 406s - loss: 0.0794 - accuracy: 0.9925 - binary_accuracy: 0.9711 - auc: 0.9404 - precision: 0.6472 - recall: 0.5190\n",
            "Test ['loss', 'accuracy', 'binary_accuracy', 'auc', 'precision', 'recall']: [0.07936915010213852, 0.9925130605697632, 0.9711466431617737, 0.9403983950614929, 0.6471701264381409, 0.5189681053161621]\n",
            "998/998 - 203s - loss: 0.0569 - accuracy: 0.9888 - binary_accuracy: 0.9791 - auc: 0.9544 - precision: 0.8337 - recall: 0.5413\n",
            "Validation ['loss', 'accuracy', 'binary_accuracy', 'auc', 'precision', 'recall']: [0.056940242648124695, 0.9887827038764954, 0.9790643453598022, 0.9544435143470764, 0.8336599469184875, 0.5412779450416565]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3LVtWu48ynJw"
      },
      "source": [
        "##Training /content/drive/MyDrive/HLT/checkpoint_20210920_195726/distil-bert . . .<br/>\n",
        "**Test** <br/> \n",
        "'loss': 0.07895965129137039 <br/> \n",
        "'accuracy': 0.991387665271759 <br/> \n",
        "'binary_accuracy': 0.9713393449783325 <br/> \n",
        "'auc': 0.941145122051239 <br/> \n",
        "'precision': 0.6544442772865295 <br/> \n",
        "'recall': 0.5108980536460876 <br/> \n",
        "<br/>\n",
        "**Validation** <br/>\n",
        "'loss': 0.05693058669567108 <br/>\n",
        "'accuracy': 0.9888767004013062 <br/>\n",
        "'binary_accuracy': 0.9790856242179871 <br/>\n",
        "'auc': 0.9567497372627258 <br/>\n",
        "'precision': 0.831497073173523 <br/>\n",
        "'recall': 0.5441051721572876] <br/>\n",
        "\n",
        "##Finetuning /content/drive/MyDrive/HLT/checkpoint_20210920_195726/ . . . <br/>\n",
        "**Test** <br/>\n",
        "'loss': 0.07936915010213852  <br/>\n",
        "'accuracy': 0.9925130605697632  <br/>\n",
        "'binary_accuracy': 0.9711466431617737  <br/>\n",
        "'auc': 0.9403983950614929  <br/>\n",
        "'precision': 0.6471701264381409  <br/>\n",
        "'recall': 0.5189681053161621  <br/>\n",
        "\n",
        "**Validation** <br/>\n",
        "'loss': 0.056940242648124695 <br/>\n",
        "'accuracy': 0.9887827038764954 <br/>\n",
        "'binary_accuracy': 0.9790643453598022 <br/>\n",
        "'auc': 0.9544435143470764 <br/>\n",
        "'precision': 0.8336599469184875 <br/> \n",
        "'recall': 0.5412779450416565 <br/>"
      ]
    }
  ]
}