3_RANK_QA_on_test_val.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "RANK_QA_on_test/val.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "dff3147983954d3cb92265513e10040e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_93e9a24fa3b34763acce3db15740c529",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_3a184dab92ab48a5a8d57db24a27f638",
              "IPY_MODEL_5ec968c3dcdd4a9f89fcc00da38dbc67"
            ]
          }
        },
        "93e9a24fa3b34763acce3db15740c529": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3a184dab92ab48a5a8d57db24a27f638": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_5ac916a147f74b3497f0fbee972c2ca2",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 231508,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 231508,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_95c2a9e0fdc64648a4532b4cc7fa96d8"
          }
        },
        "5ec968c3dcdd4a9f89fcc00da38dbc67": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_60599110acfc41679b660803136a86d2",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 226k/226k [00:16&lt;00:00, 13.8kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8a27af5afae04222a7888dc37b120259"
          }
        },
        "5ac916a147f74b3497f0fbee972c2ca2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "95c2a9e0fdc64648a4532b4cc7fa96d8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "60599110acfc41679b660803136a86d2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8a27af5afae04222a7888dc37b120259": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0c80f92ad91c47028840da50ae4ddbaa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_2fb6549c74eb4456b0d3f572bcbded12",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_bc055d972afb4c9a9b0a9985496d7d6d",
              "IPY_MODEL_2e3b2b69ccc547428a56a7457419e772"
            ]
          }
        },
        "2fb6549c74eb4456b0d3f572bcbded12": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "bc055d972afb4c9a9b0a9985496d7d6d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_657ba682cb644f0dac230dedd3a795ba",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 466062,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 466062,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_587edffd8d3e4f0ea17c8aacf17bcff8"
          }
        },
        "2e3b2b69ccc547428a56a7457419e772": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_58919a7ecac5414ab1d860d3d4126777",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 455k/455k [00:01&lt;00:00, 247kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d3735d36dcc24a6eb545836aca7710e2"
          }
        },
        "657ba682cb644f0dac230dedd3a795ba": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "587edffd8d3e4f0ea17c8aacf17bcff8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "58919a7ecac5414ab1d860d3d4126777": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d3735d36dcc24a6eb545836aca7710e2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0a904aaf26a741639d2ffbc841319e35": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_381b7406aeba4e498fe1871fa7e58164",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_626786d2ab9244fb9863c0e9bf260afc",
              "IPY_MODEL_51d5d29c6e654f7cb78709bb46a54f9e"
            ]
          }
        },
        "381b7406aeba4e498fe1871fa7e58164": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "626786d2ab9244fb9863c0e9bf260afc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_f01c58e12b4c48d1bae2082def697893",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 28,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 28,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_73c5fa6a29a3424eb27bdbdd2d6e4f5b"
          }
        },
        "51d5d29c6e654f7cb78709bb46a54f9e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_b0f2497076c84927885a4a62ee95bab4",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 28.0/28.0 [00:13&lt;00:00, 2.03B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5151043517d44d7b8d4e032e0bb822eb"
          }
        },
        "f01c58e12b4c48d1bae2082def697893": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "73c5fa6a29a3424eb27bdbdd2d6e4f5b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b0f2497076c84927885a4a62ee95bab4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5151043517d44d7b8d4e032e0bb822eb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "27beaca7419e4e5280e6afb49eba1243": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_c0ec69a03b544230b7db2bcdcf83d9e0",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_3affe25048824f4983187286dcd0ff78",
              "IPY_MODEL_84fbd9684b5d4375aee5c4557da5de75"
            ]
          }
        },
        "c0ec69a03b544230b7db2bcdcf83d9e0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3affe25048824f4983187286dcd0ff78": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_2f409ff9f16846e1afa5f9c259c6a771",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 483,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 483,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7e7f2608a25c458e8e4343ba1203267a"
          }
        },
        "84fbd9684b5d4375aee5c4557da5de75": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_20989274e65647288d8fa377065c9f75",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 483/483 [00:00&lt;00:00, 1.70kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_68d8e70df47c4a8b94a61d92e8fa2d6b"
          }
        },
        "2f409ff9f16846e1afa5f9c259c6a771": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7e7f2608a25c458e8e4343ba1203267a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "20989274e65647288d8fa377065c9f75": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "68d8e70df47c4a8b94a61d92e8fa2d6b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "!git clone https://github.com/WorkInTheDark/FairytaleQA_QAG_System.git\n",
        "%cd ./FairytaleQA_QAG_System/"
      ],
      "metadata": {
        "id": "EV767Zhg_gDa"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Save Ranking Model into PATH_TO_RANK"
      ],
      "metadata": {
        "id": "8VfAACOkqKqe"
      }
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WfZbnn24M03h"
      },
      "source": [
        "!pip install datasets\n",
        "!pip install tqdm==4.43.0\n",
        "!pip install transformers"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7ry0XlOqTmdB"
      },
      "source": [
        "from datasets import load_dataset\n",
        "from pathlib import Path"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lFyPsXelTDIb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 217,
          "referenced_widgets": [
            "dff3147983954d3cb92265513e10040e",
            "93e9a24fa3b34763acce3db15740c529",
            "3a184dab92ab48a5a8d57db24a27f638",
            "5ec968c3dcdd4a9f89fcc00da38dbc67",
            "5ac916a147f74b3497f0fbee972c2ca2",
            "95c2a9e0fdc64648a4532b4cc7fa96d8",
            "60599110acfc41679b660803136a86d2",
            "8a27af5afae04222a7888dc37b120259",
            "0c80f92ad91c47028840da50ae4ddbaa",
            "2fb6549c74eb4456b0d3f572bcbded12",
            "bc055d972afb4c9a9b0a9985496d7d6d",
            "2e3b2b69ccc547428a56a7457419e772",
            "657ba682cb644f0dac230dedd3a795ba",
            "587edffd8d3e4f0ea17c8aacf17bcff8",
            "58919a7ecac5414ab1d860d3d4126777",
            "d3735d36dcc24a6eb545836aca7710e2",
            "0a904aaf26a741639d2ffbc841319e35",
            "381b7406aeba4e498fe1871fa7e58164",
            "626786d2ab9244fb9863c0e9bf260afc",
            "51d5d29c6e654f7cb78709bb46a54f9e",
            "f01c58e12b4c48d1bae2082def697893",
            "73c5fa6a29a3424eb27bdbdd2d6e4f5b",
            "b0f2497076c84927885a4a62ee95bab4",
            "5151043517d44d7b8d4e032e0bb822eb",
            "27beaca7419e4e5280e6afb49eba1243",
            "c0ec69a03b544230b7db2bcdcf83d9e0",
            "3affe25048824f4983187286dcd0ff78",
            "84fbd9684b5d4375aee5c4557da5de75",
            "2f409ff9f16846e1afa5f9c259c6a771",
            "7e7f2608a25c458e8e4343ba1203267a",
            "20989274e65647288d8fa377065c9f75",
            "68d8e70df47c4a8b94a61d92e8fa2d6b"
          ]
        },
        "outputId": "88fb936a-5078-444f-9fe6-cb0fc6523637"
      },
      "source": [
        "from transformers import DistilBertTokenizerFast\n",
        "from transformers import DistilBertForSequenceClassification, Trainer, TrainingArguments\n",
        "tokenizer = DistilBertTokenizerFast.from_pretrained('distilbert-base-uncased')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "dff3147983954d3cb92265513e10040e",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=231508.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "0c80f92ad91c47028840da50ae4ddbaa",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=466062.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "0a904aaf26a741639d2ffbc841319e35",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=28.0, style=ProgressStyle(description_w…"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "27beaca7419e4e5280e6afb49eba1243",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=483.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9ZKGX4isTxne"
      },
      "source": [
        "\n",
        "training_args = TrainingArguments(\n",
        "    output_dir='./results',          # output directory\n",
        "    learning_rate=1.541663609697611e-05,\n",
        "    num_train_epochs=4,       # total number of training epochs\n",
        "    per_device_train_batch_size=1,  # batch size per device during training\n",
        "    per_device_eval_batch_size=1,   # batch size for evaluation\n",
        "    warmup_steps=500,                # number of warmup steps for learning rate scheduler\n",
        "    weight_decay=0.01,               # strength of weight decay\n",
        "    logging_dir='./logs',            # directory for storing logs\n",
        "    logging_steps=30000,\n",
        "    save_steps=100000,\n",
        "    seed=15,\n",
        "\n",
        ")\n",
        "\n",
        "model = DistilBertForSequenceClassification.from_pretrained(\"PATH_TO_RANK/RANKING_QA_DistilBert_QAC_lr1e-5_e4_seed15_b1_uneven_distributed\", local_files_only=True)\n",
        "\n",
        "\n",
        "trainer = Trainer(\n",
        "    model=model,                         # the instantiated 🤗 Transformers model to be trained\n",
        "    args=training_args,                  # training arguments, defined above\n",
        "    # train_dataset=train_dataset,         # training dataset\n",
        "    # eval_dataset=val_dataset             # evaluation dataset\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Uzr0y1ucTlZg"
      },
      "source": [
        "import torch\n",
        "\n",
        "class RankingDataset(torch.utils.data.Dataset):\n",
        "    def __init__(self, encodings, labels):\n",
        "        self.encodings = encodings\n",
        "        self.labels = labels\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
        "        item['labels'] = torch.tensor(self.labels[idx])\n",
        "        return item\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.labels)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DXpYsVg3uC_S"
      },
      "source": [
        "test data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Se_X7Cu1kkHj"
      },
      "source": [
        "!mkdir ./FairytaleQA_QAG_System/result\n",
        "!mkdir ./FairytaleQA_QAG_System/result/test"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "save_directory = \"./FairytaleQA_QAG_System/result/test\""
      ],
      "metadata": {
        "id": "EsGvhQjL_-fj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eg0I52wmOEFr"
      },
      "source": [
        "import glob \n",
        "\n",
        "generation_results = []\n",
        "\n",
        "total_generated_results = 0\n",
        "\n",
        "\n",
        "cnt_book = 0\n",
        "\n",
        "book_directory = glob.glob(\"./FairytaleQA_QAG_System/data/generated_qa/test/*.txt\", recursive=True)\n",
        "\n",
        "\n",
        "for each_book in book_directory:\n",
        "  key = each_book.split('/')[-1].replace('.txt', '').replace('generated_qa_stories-', '').lower()\n",
        "\n",
        "  # this to be a section-key dictionary\n",
        "  generation_answers = []\n",
        "  generation_data_for_ranking = []\n",
        "\n",
        "  lines_generate = open(each_book, 'r').readlines()\n",
        "  \n",
        "  ## for i in lines_generate:\n",
        "  for i in lines_generate:\n",
        "    # each line should be A </s> C </s> Q\n",
        "    \n",
        "    generation_answers.append( i.replace('\\n', '')  )\n",
        "    \n",
        "    question = i.replace('\\n', '').split('</s>')[-1].strip()\n",
        "    answer = i.replace('\\n', '').split('</s>')[0].strip()\n",
        "    section = i.replace('\\n', '').split('</s>')[1].strip()\n",
        "\n",
        "    # Q + A + C\n",
        "    generation_data_for_ranking.append( question + ' <SEP> ' + answer  + ' <SEP> ' + section)\n",
        "\n",
        "\n",
        "    total_generated_results += 1\n",
        "\n",
        "\n",
        "\n",
        "  # create the evaluation dataset for the ranking model, and run through the model\n",
        "  \n",
        "  test_encodings = tokenizer(generation_data_for_ranking, truncation=True, padding=True)\n",
        "  test_labels = [1]*len(generation_data_for_ranking)\n",
        "  test_dataset = RankingDataset(test_encodings, test_labels)\n",
        "\n",
        "  predictions = trainer.predict(test_dataset)\n",
        "  \n",
        "  assert len(predictions.predictions) == len(generation_answers)\n",
        "\n",
        "  # Here you may store all ranked QA-pairs along with the score into a file\n",
        "  # Or just directly sort by the values and select the top-N ones\n",
        "\n",
        "  \n",
        "  f1 = open(save_directory + '/ranked_generated_qa_stories-' + key + \".txt\", 'w')\n",
        "  \n",
        "  for i in range(len(predictions.predictions)):\n",
        "    predicted =  predictions.predictions[i][1]\n",
        "    f1.write( generation_answers[i] + '<SEP>' + str(predicted) + '\\n' )\n",
        "  \n",
        "  f1.close()\n",
        "\n",
        "\n",
        "  cnt_book += 1"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "exvhVgo4uEkJ"
      },
      "source": [
        "val data"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!mkdir ./FairytaleQA_QAG_System/result/val"
      ],
      "metadata": {
        "id": "0kij-i-RAoKk"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4d639sBvkwTK"
      },
      "source": [
        "save_directory = \"./FairytaleQA_QAG_System/result/val\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ERC611z05UhH"
      },
      "source": [
        "import glob \n",
        "\n",
        "generation_results = []\n",
        "\n",
        "total_generated_results = 0\n",
        "\n",
        "\n",
        "cnt_book = 0\n",
        "\n",
        "book_directory = glob.glob(\"./FairytaleQA_QAG_System/data/generated_qa/val/*.txt\", recursive=True)\n",
        "\n",
        "\n",
        "# for key in val_name_set:\n",
        "# for key in test_name_set:\n",
        "for each_book in book_directory:\n",
        "  key = each_book.split('/')[-1].replace('.txt', '').replace('generated_qa_stories-', '').lower()\n",
        "\n",
        "  # this to be a section-key dictionary\n",
        "  generation_answers = []\n",
        "  generation_data_for_ranking = []\n",
        "\n",
        "  lines_generate = open(each_book, 'r').readlines()\n",
        "\n",
        "  for i in lines_generate:\n",
        "\n",
        "    generation_answers.append( i.replace('\\n', '')  )\n",
        "    \n",
        "    question = i.replace('\\n', '').split('</s>')[-1].strip()\n",
        "    answer = i.replace('\\n', '').split('</s>')[0].strip()\n",
        "    section = i.replace('\\n', '').split('</s>')[1].strip()\n",
        "\n",
        "\n",
        "    # Q + A + C\n",
        "    generation_data_for_ranking.append( question + ' <SEP> ' + answer + ' <SEP> ' + section)\n",
        "\n",
        "    total_generated_results += 1\n",
        "\n",
        "\n",
        "\n",
        "  # create the evaluation dataset for the ranking model, and run through the model\n",
        "  \n",
        "  test_encodings = tokenizer(generation_data_for_ranking, truncation=True, padding=True)\n",
        "  test_labels = [1]*len(generation_data_for_ranking)\n",
        "  test_dataset = RankingDataset(test_encodings, test_labels)\n",
        "\n",
        "  predictions = trainer.predict(test_dataset)\n",
        "  \n",
        "  assert len(predictions.predictions) == len(generation_answers)\n",
        "\n",
        "  # Here you may store all ranked QA-pairs along with the score into a file\n",
        "  # Or just directly sort by the values and select the top-N ones\n",
        "\n",
        "  \n",
        "  f1 = open(save_directory + '/ranked_generated_qa_stories-' + key + \".txt\", 'w')\n",
        "  \n",
        "  for i in range(len(predictions.predictions)):\n",
        "    predicted =  predictions.predictions[i][1]\n",
        "    f1.write( generation_answers[i] + '<SEP>' + str(predicted) + '\\n' )\n",
        "  \n",
        "  f1.close()\n",
        "\n",
        "\n",
        "  cnt_book += 1"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}