MLLU_Final_Project.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "MLLU_Final_Project_V2.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "TPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "ae9c13e33d1d491e87261407a5077e81": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_b04e79e6b95d42e09c31a16bc0fe55b5",
              "IPY_MODEL_7025584292954e6e9d19ca210b817846",
              "IPY_MODEL_34720ea1d70f426d87a66713f13f9b5e"
            ],
            "layout": "IPY_MODEL_59ac794e63fe4ee1ae48c37c89afb5e1"
          }
        },
        "b04e79e6b95d42e09c31a16bc0fe55b5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_93a3d2193a484277a6fec24a0fe53a8b",
            "placeholder": "​",
            "style": "IPY_MODEL_c98b27e6c58e499aafb088265d2a5905",
            "value": "Downloading: 100%"
          }
        },
        "7025584292954e6e9d19ca210b817846": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ccc663fd8408406b9b0239e37718fde3",
            "max": 5069051,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_205fac7b18d94208991fb913ea61b314",
            "value": 5069051
          }
        },
        "34720ea1d70f426d87a66713f13f9b5e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_25bfb13ee0ba47aca9a728c8254ad6bd",
            "placeholder": "​",
            "style": "IPY_MODEL_8fcfb1e0aa754b458a33459e067a8923",
            "value": " 4.83M/4.83M [00:00&lt;00:00, 9.35MB/s]"
          }
        },
        "59ac794e63fe4ee1ae48c37c89afb5e1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "93a3d2193a484277a6fec24a0fe53a8b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c98b27e6c58e499aafb088265d2a5905": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ccc663fd8408406b9b0239e37718fde3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "205fac7b18d94208991fb913ea61b314": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "25bfb13ee0ba47aca9a728c8254ad6bd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8fcfb1e0aa754b458a33459e067a8923": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "df4d7db67c0c44a98e5b5dab66a19438": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_1fd38d5f20154727ad12035f017fa430",
              "IPY_MODEL_eedaf2495d664ca1acab4ff9b29d9ef8",
              "IPY_MODEL_8fcac9b255bf4b33adcc01da5aaeb57d"
            ],
            "layout": "IPY_MODEL_b3b20ab1b9d64a5facf0cd561dea0c2d"
          }
        },
        "1fd38d5f20154727ad12035f017fa430": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2cc9ef1d2e54424894da9d876cb08ddc",
            "placeholder": "​",
            "style": "IPY_MODEL_49da1cdca80d4a28ada6937bff1cf799",
            "value": "Downloading: 100%"
          }
        },
        "eedaf2495d664ca1acab4ff9b29d9ef8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5dda7251ad7a485fb27fcc84d312ae04",
            "max": 615,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_0ff228fb0840489390dbd35314039d2a",
            "value": 615
          }
        },
        "8fcac9b255bf4b33adcc01da5aaeb57d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bb046b5e8beb413c8c986b7953f34f10",
            "placeholder": "​",
            "style": "IPY_MODEL_44c2e5696ad140e6b45507af2abf2b88",
            "value": " 615/615 [00:00&lt;00:00, 18.7kB/s]"
          }
        },
        "b3b20ab1b9d64a5facf0cd561dea0c2d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2cc9ef1d2e54424894da9d876cb08ddc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "49da1cdca80d4a28ada6937bff1cf799": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5dda7251ad7a485fb27fcc84d312ae04": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0ff228fb0840489390dbd35314039d2a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "bb046b5e8beb413c8c986b7953f34f10": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "44c2e5696ad140e6b45507af2abf2b88": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# Initial Setup"
      ],
      "metadata": {
        "id": "SxQCg2ImehX2"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "assert os.environ['COLAB_TPU_ADDR'], 'Make sure to select TPU from Edit > Notebook settings > Hardware accelerator'"
      ],
      "metadata": {
        "id": "TE2KgG6Mr629"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install cloud-tpu-client==0.10 torch==1.11.0 https://storage.googleapis.com/tpu-pytorch/wheels/colab/torch_xla-1.11-cp37-cp37m-linux_x86_64.whl"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XMI4tWEFnTPD",
        "outputId": "f9a0e107-810a-4f6f-84c5-d7db3cdfb884"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting torch-xla==1.11\n",
            "  Downloading https://storage.googleapis.com/tpu-pytorch/wheels/colab/torch_xla-1.11-cp37-cp37m-linux_x86_64.whl (152.9 MB)\n",
            "\u001b[K     |████████████████████████████████| 152.9 MB 34 kB/s \n",
            "\u001b[?25hCollecting cloud-tpu-client==0.10\n",
            "  Downloading cloud_tpu_client-0.10-py3-none-any.whl (7.4 kB)\n",
            "Requirement already satisfied: torch==1.11.0 in /usr/local/lib/python3.7/dist-packages (1.11.0+cu113)\n",
            "Collecting google-api-python-client==1.8.0\n",
            "  Downloading google_api_python_client-1.8.0-py3-none-any.whl (57 kB)\n",
            "\u001b[K     |████████████████████████████████| 57 kB 2.5 MB/s \n",
            "\u001b[?25hRequirement already satisfied: oauth2client in /usr/local/lib/python3.7/dist-packages (from cloud-tpu-client==0.10) (4.1.3)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch==1.11.0) (4.2.0)\n",
            "Requirement already satisfied: google-api-core<2dev,>=1.13.0 in /usr/local/lib/python3.7/dist-packages (from google-api-python-client==1.8.0->cloud-tpu-client==0.10) (1.31.5)\n",
            "Requirement already satisfied: httplib2<1dev,>=0.9.2 in /usr/local/lib/python3.7/dist-packages (from google-api-python-client==1.8.0->cloud-tpu-client==0.10) (0.17.4)\n",
            "Requirement already satisfied: six<2dev,>=1.6.1 in /usr/local/lib/python3.7/dist-packages (from google-api-python-client==1.8.0->cloud-tpu-client==0.10) (1.15.0)\n",
            "Requirement already satisfied: google-auth>=1.4.1 in /usr/local/lib/python3.7/dist-packages (from google-api-python-client==1.8.0->cloud-tpu-client==0.10) (1.35.0)\n",
            "Requirement already satisfied: google-auth-httplib2>=0.0.3 in /usr/local/lib/python3.7/dist-packages (from google-api-python-client==1.8.0->cloud-tpu-client==0.10) (0.0.4)\n",
            "Requirement already satisfied: uritemplate<4dev,>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from google-api-python-client==1.8.0->cloud-tpu-client==0.10) (3.0.1)\n",
            "Requirement already satisfied: requests<3.0.0dev,>=2.18.0 in /usr/local/lib/python3.7/dist-packages (from google-api-core<2dev,>=1.13.0->google-api-python-client==1.8.0->cloud-tpu-client==0.10) (2.23.0)\n",
            "Requirement already satisfied: setuptools>=40.3.0 in /usr/local/lib/python3.7/dist-packages (from google-api-core<2dev,>=1.13.0->google-api-python-client==1.8.0->cloud-tpu-client==0.10) (57.4.0)\n",
            "Requirement already satisfied: packaging>=14.3 in /usr/local/lib/python3.7/dist-packages (from google-api-core<2dev,>=1.13.0->google-api-python-client==1.8.0->cloud-tpu-client==0.10) (21.3)\n",
            "Requirement already satisfied: protobuf>=3.12.0 in /usr/local/lib/python3.7/dist-packages (from google-api-core<2dev,>=1.13.0->google-api-python-client==1.8.0->cloud-tpu-client==0.10) (3.17.3)\n",
            "Requirement already satisfied: pytz in /usr/local/lib/python3.7/dist-packages (from google-api-core<2dev,>=1.13.0->google-api-python-client==1.8.0->cloud-tpu-client==0.10) (2022.1)\n",
            "Requirement already satisfied: googleapis-common-protos<2.0dev,>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from google-api-core<2dev,>=1.13.0->google-api-python-client==1.8.0->cloud-tpu-client==0.10) (1.56.0)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth>=1.4.1->google-api-python-client==1.8.0->cloud-tpu-client==0.10) (0.2.8)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth>=1.4.1->google-api-python-client==1.8.0->cloud-tpu-client==0.10) (4.8)\n",
            "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth>=1.4.1->google-api-python-client==1.8.0->cloud-tpu-client==0.10) (4.2.4)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=14.3->google-api-core<2dev,>=1.13.0->google-api-python-client==1.8.0->cloud-tpu-client==0.10) (3.0.8)\n",
            "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth>=1.4.1->google-api-python-client==1.8.0->cloud-tpu-client==0.10) (0.4.8)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0dev,>=2.18.0->google-api-core<2dev,>=1.13.0->google-api-python-client==1.8.0->cloud-tpu-client==0.10) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0dev,>=2.18.0->google-api-core<2dev,>=1.13.0->google-api-python-client==1.8.0->cloud-tpu-client==0.10) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0dev,>=2.18.0->google-api-core<2dev,>=1.13.0->google-api-python-client==1.8.0->cloud-tpu-client==0.10) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0dev,>=2.18.0->google-api-core<2dev,>=1.13.0->google-api-python-client==1.8.0->cloud-tpu-client==0.10) (2021.10.8)\n",
            "Installing collected packages: google-api-python-client, torch-xla, cloud-tpu-client\n",
            "  Attempting uninstall: google-api-python-client\n",
            "    Found existing installation: google-api-python-client 1.12.11\n",
            "    Uninstalling google-api-python-client-1.12.11:\n",
            "      Successfully uninstalled google-api-python-client-1.12.11\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "earthengine-api 0.1.307 requires google-api-python-client<2,>=1.12.1, but you have google-api-python-client 1.8.0 which is incompatible.\u001b[0m\n",
            "Successfully installed cloud-tpu-client-0.10 google-api-python-client-1.8.0 torch-xla-1.11\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install transformers\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xf_JN6zn-_2M",
        "outputId": "cfbcb325-c8cf-4de2-efdb-cb8736000e16"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting transformers\n",
            "  Downloading transformers-4.19.0-py3-none-any.whl (4.2 MB)\n",
            "\u001b[K     |████████████████████████████████| 4.2 MB 5.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.64.0)\n",
            "Collecting tokenizers!=0.11.3,<0.13,>=0.11.1\n",
            "  Downloading tokenizers-0.12.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (6.6 MB)\n",
            "\u001b[K     |████████████████████████████████| 6.6 MB 47.5 MB/s \n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.6.0)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (21.3)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.21.6)\n",
            "Collecting pyyaml>=5.1\n",
            "  Downloading PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (596 kB)\n",
            "\u001b[K     |████████████████████████████████| 596 kB 53.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.11.3)\n",
            "Collecting huggingface-hub<1.0,>=0.1.0\n",
            "  Downloading huggingface_hub-0.6.0-py3-none-any.whl (84 kB)\n",
            "\u001b[K     |████████████████████████████████| 84 kB 3.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0,>=0.1.0->transformers) (4.2.0)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers) (3.0.8)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.8.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2021.10.8)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Installing collected packages: pyyaml, tokenizers, huggingface-hub, transformers\n",
            "  Attempting uninstall: pyyaml\n",
            "    Found existing installation: PyYAML 3.13\n",
            "    Uninstalling PyYAML-3.13:\n",
            "      Successfully uninstalled PyYAML-3.13\n",
            "Successfully installed huggingface-hub-0.6.0 pyyaml-6.0 tokenizers-0.12.1 transformers-4.19.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install Sentencepiece\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bxNosiUr_2Rz",
        "outputId": "1b495cc7-b387-43c2-f3c0-8e7275697cc9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting Sentencepiece\n",
            "  Downloading sentencepiece-0.1.96-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.2 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.2 MB 4.0 MB/s \n",
            "\u001b[?25hInstalling collected packages: Sentencepiece\n",
            "Successfully installed Sentencepiece-0.1.96\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "import torch_xla\n",
        "import torch_xla.core.xla_model as xm"
      ],
      "metadata": {
        "id": "NPvB-VNHel1C",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "acb64834-4d37-446e-96a9-9cda0f68c297"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:root:Waiting for TPU to be start up with version pytorch-1.11...\n",
            "WARNING:root:Waiting for TPU to be start up with version pytorch-1.11...\n",
            "WARNING:root:TPU has started up successfully with version pytorch-1.11\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "device = xm.xla_device()"
      ],
      "metadata": {
        "id": "oBqFUllHsZ9y"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "import torch.nn.functional as F\n",
        "from torch.utils.data import Dataset, DataLoader\n",
        "\n",
        "# set a seed value\n",
        "torch.manual_seed(555)\n",
        "\n",
        "from sklearn.utils import shuffle\n",
        "from sklearn.metrics import roc_auc_score, accuracy_score\n",
        "\n",
        "import transformers\n",
        "from transformers import BertTokenizer, BertForSequenceClassification \n",
        "from transformers import XLMRobertaTokenizer, XLMRobertaForSequenceClassification, XLMRobertaModel\n",
        "from transformers import AdamW"
      ],
      "metadata": {
        "id": "FR6D0JXJfBeh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "rTokenizer = XLMRobertaTokenizer.from_pretrained('xlm-roberta-base')\n"
      ],
      "metadata": {
        "id": "LhfpG51UfqD-",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 109,
          "referenced_widgets": [
            "ae9c13e33d1d491e87261407a5077e81",
            "b04e79e6b95d42e09c31a16bc0fe55b5",
            "7025584292954e6e9d19ca210b817846",
            "34720ea1d70f426d87a66713f13f9b5e",
            "59ac794e63fe4ee1ae48c37c89afb5e1",
            "93a3d2193a484277a6fec24a0fe53a8b",
            "c98b27e6c58e499aafb088265d2a5905",
            "ccc663fd8408406b9b0239e37718fde3",
            "205fac7b18d94208991fb913ea61b314",
            "25bfb13ee0ba47aca9a728c8254ad6bd",
            "8fcfb1e0aa754b458a33459e067a8923",
            "df4d7db67c0c44a98e5b5dab66a19438",
            "1fd38d5f20154727ad12035f017fa430",
            "eedaf2495d664ca1acab4ff9b29d9ef8",
            "8fcac9b255bf4b33adcc01da5aaeb57d",
            "b3b20ab1b9d64a5facf0cd561dea0c2d",
            "2cc9ef1d2e54424894da9d876cb08ddc",
            "49da1cdca80d4a28ada6937bff1cf799",
            "5dda7251ad7a485fb27fcc84d312ae04",
            "0ff228fb0840489390dbd35314039d2a",
            "bb046b5e8beb413c8c986b7953f34f10",
            "44c2e5696ad140e6b45507af2abf2b88"
          ]
        },
        "outputId": "e9124dab-fdb8-42a9-8d56-711c6cc29787"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/4.83M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "ae9c13e33d1d491e87261407a5077e81"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/615 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "df4d7db67c0c44a98e5b5dab66a19438"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(rTokenizer.vocab_size)"
      ],
      "metadata": {
        "id": "Z582Kx1VERNO",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5427dea1-c1f9-43f7-cbb0-a87da4e104c0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "250002\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Data"
      ],
      "metadata": {
        "id": "oY40ZxVsOx0N"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Import\n"
      ],
      "metadata": {
        "id": "kId-9gUedN3R"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!wget --no-check-certificate 'https://docs.google.com/uc?export=download&id=1RA8uE4DRgEV2iHx2kLnPvUgXWZ3gTJM1' -O 'chinese.csv'\n",
        "# !wget --no-check-certificate 'https://docs.google.com/uc?export=download&id=1f2He-xBuCjW4BU1BqWKXivCE1_RhSGha' -O 'english.csv'\n",
        "!wget --no-check-certificate 'https://docs.google.com/uc?export=download&id=13OYXerLI7O3kGfanfoBlcEFqvGzi9Q0z' -O 'spanish.csv'\n",
        "!wget --no-check-certificate 'https://docs.google.com/uc?export=download&id=1t-cPh9CBtKZYTDzunGQhsFbIYWfyRGeQ' -O 'marathi.csv'\n",
        "!wget --no-check-certificate 'https://docs.google.com/uc?export=download&id=1Jrpm7fMu2nb0axD0iiJ8mgDoyiVLvixc' -O 'urdu.tsv'\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "39jBZZDuma7a",
        "outputId": "2fabfa18-1f78-4a2b-a78d-488a9261d336"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2022-05-13 14:45:41--  https://docs.google.com/uc?export=download&id=1RA8uE4DRgEV2iHx2kLnPvUgXWZ3gTJM1\n",
            "Resolving docs.google.com (docs.google.com)... 173.194.192.100, 173.194.192.139, 173.194.192.138, ...\n",
            "Connecting to docs.google.com (docs.google.com)|173.194.192.100|:443... connected.\n",
            "HTTP request sent, awaiting response... 303 See Other\n",
            "Location: https://doc-14-6k-docs.googleusercontent.com/docs/securesc/ha0ro937gcuc7l7deffksulhg5h7mbp1/htv1pivoeto0ep64gatpapeppv85juhg/1652453100000/08219369989881812643/*/1RA8uE4DRgEV2iHx2kLnPvUgXWZ3gTJM1?e=download [following]\n",
            "Warning: wildcards not supported in HTTP.\n",
            "--2022-05-13 14:45:44--  https://doc-14-6k-docs.googleusercontent.com/docs/securesc/ha0ro937gcuc7l7deffksulhg5h7mbp1/htv1pivoeto0ep64gatpapeppv85juhg/1652453100000/08219369989881812643/*/1RA8uE4DRgEV2iHx2kLnPvUgXWZ3gTJM1?e=download\n",
            "Resolving doc-14-6k-docs.googleusercontent.com (doc-14-6k-docs.googleusercontent.com)... 108.177.111.132, 2607:f8b0:4001:c07::84\n",
            "Connecting to doc-14-6k-docs.googleusercontent.com (doc-14-6k-docs.googleusercontent.com)|108.177.111.132|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 19699818 (19M) [text/csv]\n",
            "Saving to: ‘chinese.csv’\n",
            "\n",
            "chinese.csv         100%[===================>]  18.79M  62.4MB/s    in 0.3s    \n",
            "\n",
            "2022-05-13 14:45:44 (62.4 MB/s) - ‘chinese.csv’ saved [19699818/19699818]\n",
            "\n",
            "--2022-05-13 14:45:44--  https://docs.google.com/uc?export=download&id=13OYXerLI7O3kGfanfoBlcEFqvGzi9Q0z\n",
            "Resolving docs.google.com (docs.google.com)... 173.194.192.101, 173.194.192.100, 173.194.192.113, ...\n",
            "Connecting to docs.google.com (docs.google.com)|173.194.192.101|:443... connected.\n",
            "HTTP request sent, awaiting response... 303 See Other\n",
            "Location: https://doc-00-6k-docs.googleusercontent.com/docs/securesc/ha0ro937gcuc7l7deffksulhg5h7mbp1/0qee6o2rjeguoonmvpd7l14b977bron6/1652453100000/08219369989881812643/*/13OYXerLI7O3kGfanfoBlcEFqvGzi9Q0z?e=download [following]\n",
            "Warning: wildcards not supported in HTTP.\n",
            "--2022-05-13 14:45:46--  https://doc-00-6k-docs.googleusercontent.com/docs/securesc/ha0ro937gcuc7l7deffksulhg5h7mbp1/0qee6o2rjeguoonmvpd7l14b977bron6/1652453100000/08219369989881812643/*/13OYXerLI7O3kGfanfoBlcEFqvGzi9Q0z?e=download\n",
            "Resolving doc-00-6k-docs.googleusercontent.com (doc-00-6k-docs.googleusercontent.com)... 108.177.111.132, 2607:f8b0:4001:c07::84\n",
            "Connecting to doc-00-6k-docs.googleusercontent.com (doc-00-6k-docs.googleusercontent.com)|108.177.111.132|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 1537413 (1.5M) [text/csv]\n",
            "Saving to: ‘spanish.csv’\n",
            "\n",
            "spanish.csv         100%[===================>]   1.47M  --.-KB/s    in 0.009s  \n",
            "\n",
            "2022-05-13 14:45:46 (167 MB/s) - ‘spanish.csv’ saved [1537413/1537413]\n",
            "\n",
            "--2022-05-13 14:45:46--  https://docs.google.com/uc?export=download&id=1t-cPh9CBtKZYTDzunGQhsFbIYWfyRGeQ\n",
            "Resolving docs.google.com (docs.google.com)... 173.194.192.138, 173.194.192.101, 173.194.192.100, ...\n",
            "Connecting to docs.google.com (docs.google.com)|173.194.192.138|:443... connected.\n",
            "HTTP request sent, awaiting response... 303 See Other\n",
            "Location: https://doc-00-6k-docs.googleusercontent.com/docs/securesc/ha0ro937gcuc7l7deffksulhg5h7mbp1/sifg66e47skhgsbi4b50pe9kdqkdfb12/1652453100000/08219369989881812643/*/1t-cPh9CBtKZYTDzunGQhsFbIYWfyRGeQ?e=download [following]\n",
            "Warning: wildcards not supported in HTTP.\n",
            "--2022-05-13 14:45:47--  https://doc-00-6k-docs.googleusercontent.com/docs/securesc/ha0ro937gcuc7l7deffksulhg5h7mbp1/sifg66e47skhgsbi4b50pe9kdqkdfb12/1652453100000/08219369989881812643/*/1t-cPh9CBtKZYTDzunGQhsFbIYWfyRGeQ?e=download\n",
            "Resolving doc-00-6k-docs.googleusercontent.com (doc-00-6k-docs.googleusercontent.com)... 108.177.111.132, 2607:f8b0:4001:c07::84\n",
            "Connecting to doc-00-6k-docs.googleusercontent.com (doc-00-6k-docs.googleusercontent.com)|108.177.111.132|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 6438448 (6.1M) [text/csv]\n",
            "Saving to: ‘marathi.csv’\n",
            "\n",
            "marathi.csv         100%[===================>]   6.14M  --.-KB/s    in 0.07s   \n",
            "\n",
            "2022-05-13 14:45:48 (82.8 MB/s) - ‘marathi.csv’ saved [6438448/6438448]\n",
            "\n",
            "--2022-05-13 14:45:48--  https://docs.google.com/uc?export=download&id=1Jrpm7fMu2nb0axD0iiJ8mgDoyiVLvixc\n",
            "Resolving docs.google.com (docs.google.com)... 173.194.192.139, 173.194.192.101, 173.194.192.102, ...\n",
            "Connecting to docs.google.com (docs.google.com)|173.194.192.139|:443... connected.\n",
            "HTTP request sent, awaiting response... 303 See Other\n",
            "Location: https://doc-00-6k-docs.googleusercontent.com/docs/securesc/ha0ro937gcuc7l7deffksulhg5h7mbp1/26p5jv1mivdp708db7jju6ull4vrcec1/1652453100000/08219369989881812643/*/1Jrpm7fMu2nb0axD0iiJ8mgDoyiVLvixc?e=download [following]\n",
            "Warning: wildcards not supported in HTTP.\n",
            "--2022-05-13 14:45:48--  https://doc-00-6k-docs.googleusercontent.com/docs/securesc/ha0ro937gcuc7l7deffksulhg5h7mbp1/26p5jv1mivdp708db7jju6ull4vrcec1/1652453100000/08219369989881812643/*/1Jrpm7fMu2nb0axD0iiJ8mgDoyiVLvixc?e=download\n",
            "Resolving doc-00-6k-docs.googleusercontent.com (doc-00-6k-docs.googleusercontent.com)... 108.177.111.132, 2607:f8b0:4001:c07::84\n",
            "Connecting to doc-00-6k-docs.googleusercontent.com (doc-00-6k-docs.googleusercontent.com)|108.177.111.132|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 152410 (149K) [text/tab-separated-values]\n",
            "Saving to: ‘urdu.tsv’\n",
            "\n",
            "urdu.tsv            100%[===================>] 148.84K  --.-KB/s    in 0.006s  \n",
            "\n",
            "2022-05-13 14:45:48 (22.4 MB/s) - ‘urdu.tsv’ saved [152410/152410]\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3Fayd6QtEn6-",
        "outputId": "699934ed-4000-4b03-f970-2246a2ed039f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "from sklearn.metrics import f1_score\n",
        "from sklearn.metrics import precision_score\n",
        "from sklearn.metrics import recall_score\n",
        "import gc"
      ],
      "metadata": {
        "id": "RTYjCWDhREZy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "urduRaw = pd.read_csv(\"urdu.tsv\", sep='\\t')\n",
        "spanishRaw = pd.read_csv(\"spanish.csv\")\n",
        "marathiRaw = pd.read_csv(\"marathi.csv\")\n",
        "chineseRaw = pd.read_csv(\"chinese.csv\")"
      ],
      "metadata": {
        "id": "mM1IsHFjRJC7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# raw data file too large for !wget. Imported from google drive\n",
        "englishRaw = pd.read_csv(\"drive/MyDrive/dataset/english.csv\", delimiter=',', header=None)"
      ],
      "metadata": {
        "id": "5E9qfXZepH-i"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Cleaning\n",
        "### Spanish"
      ],
      "metadata": {
        "id": "UwSbW2NIvhqn"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "spanish = spanishRaw[[\"text\", \"airline_sentiment\"]].copy(deep=True).sample(frac=1).reset_index(drop=True)\n",
        "spanish.rename(columns = {'airline_sentiment':'label'}, inplace = True)\n",
        "spanish = spanish[spanish['label']!=\"neutral\"]\n",
        "spanish.loc[(spanish.label == \"negative\"), \"label\"] = 0\n",
        "spanish.loc[(spanish.label == 'positive'), \"label\"] = 1\n",
        "spanish['language']='spanish'"
      ],
      "metadata": {
        "id": "CgDXPMZN_icb"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "spanish['label'].value_counts()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BnjyO4AbJ6dW",
        "outputId": "afa77793-746d-4092-bc6e-a38b96371715"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0    3769\n",
              "1    1489\n",
              "Name: label, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "val_sizeS = int(spanish.shape[0] * 0.15)\n",
        "test_sizeS = int(spanish.shape[0] * 0.15)\n",
        "test_spanish = spanish[:test_sizeS].reset_index(drop=True)\n",
        "val_spanish = spanish[test_sizeS:test_sizeS+val_sizeS].reset_index(drop=True)\n",
        "train_spanish = spanish[test_sizeS+val_sizeS:].reset_index(drop=True)"
      ],
      "metadata": {
        "id": "BSVn138IAvXk"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Urdu"
      ],
      "metadata": {
        "id": "GV5WEO6yCKzF"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "urdu=urduRaw.copy(deep=True).sample(frac=1).reset_index(drop=True)\n",
        "urdu.rename(columns = {'Tweet':'text','Class':'label'}, inplace = True)\n",
        "urdu = urdu[urdu['label'].notna()]\n",
        "urdu = urdu[urdu['label']!='O']\n",
        "urdu.loc[(urdu.label == \"N\"), \"label\"] = 0\n",
        "urdu.loc[(urdu.label == 'P'), \"label\"] = 1\n",
        "urdu['language']='urdu'"
      ],
      "metadata": {
        "id": "wzeiso8sCOI0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "urdu['label'].value_counts()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "K8y0REdrM3J_",
        "outputId": "9247a645-9e5f-4d43-ebc7-8e64e0a7face"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0    499\n",
              "1    480\n",
              "Name: label, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "val_sizeU = int(urdu.shape[0] * 0.15)\n",
        "test_sizeU = int(urdu.shape[0] * 0.15)\n",
        "test_urdu = urdu[:test_sizeU].reset_index(drop=True)\n",
        "val_urdu = urdu[test_sizeU:test_sizeU+val_sizeU].reset_index(drop=True)\n",
        "train_urdu = urdu[test_sizeU+val_sizeU:].reset_index(drop=True)"
      ],
      "metadata": {
        "id": "QcXkjXABE8NH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Marathi"
      ],
      "metadata": {
        "id": "K9GEsCadGX7s"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "marathi=marathiRaw.copy(deep=True).sample(frac=1).reset_index(drop=True)\n",
        "marathi.rename(columns = {'tweet':'text'}, inplace = True)\n",
        "marathi = marathi[marathi['label']!=0]\n",
        "marathi.loc[(marathi.label == -1), \"label\"] = 0\n",
        "marathi['language']='marathi'"
      ],
      "metadata": {
        "id": "TIgSz9cOF2SI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "marathi['label'].value_counts()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qt0w0FwNGxfG",
        "outputId": "db223407-45ae-40a3-c751-1badbdd40a8a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0    4038\n",
              "1    4038\n",
              "Name: label, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "val_sizeM = int(marathi.shape[0] * 0.15)\n",
        "test_sizeM = int(marathi.shape[0] * 0.15)\n",
        "test_marathi = marathi[:test_sizeM].reset_index(drop=True)\n",
        "val_marathi = marathi[test_sizeM:test_sizeM+val_sizeM].reset_index(drop=True)\n",
        "train_marathi = marathi[test_sizeM+val_sizeM:].reset_index(drop=True)"
      ],
      "metadata": {
        "id": "_Nx9inCnG4JL"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Chinese"
      ],
      "metadata": {
        "id": "QDRDBnUIHjps"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "chinese = chineseRaw[[\"review\", \"label\"]].copy(deep=True).sample(frac=1).reset_index(drop=True)\n",
        "chinese.rename(columns = {'review':'text'}, inplace = True)"
      ],
      "metadata": {
        "id": "GssWX_XiHiec"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "chinese['language']='chinese'"
      ],
      "metadata": {
        "id": "tSX78or_M7qZ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "chinese = chinese.sample(9000).reset_index(drop=True).copy()\n",
        "chinese['label'].value_counts()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "i1LAQ6rRIB6b",
        "outputId": "47ec1843-9b81-40f6-da0f-d2ae25b7f2f1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1    4530\n",
              "0    4470\n",
              "Name: label, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "val_sizeC = int(chinese.shape[0] * 0.15)\n",
        "test_sizeC = int(chinese.shape[0] * 0.15)\n",
        "test_chinese = chinese[:test_sizeC].reset_index(drop=True)\n",
        "val_chinese = chinese[test_sizeC:test_sizeC+val_sizeC].reset_index(drop=True)\n",
        "train_chinese = chinese[test_sizeC+val_sizeC:].reset_index(drop=True)"
      ],
      "metadata": {
        "id": "pwgqx7BOIEXt"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### English"
      ],
      "metadata": {
        "id": "DgzNIuwrIN8e"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "english = englishRaw[[5, 0]].copy(deep=True).sample(frac=1).reset_index(drop=True)\n",
        "english.rename(columns = {5:'text', 0:'label'}, inplace = True)\n",
        "english.loc[(english.label == 4), \"label\"] = 1\n",
        "english['language']='english'"
      ],
      "metadata": {
        "id": "9z4_5MJZIRe5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "english = english.sample(9000).reset_index(drop=True).copy()\n",
        "english['label'].value_counts()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "H4pPpLarI6Fx",
        "outputId": "b9bf42df-b511-46d8-c88c-f64076bff2c3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1    4536\n",
              "0    4464\n",
              "Name: label, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "val_sizeE = int(english.shape[0] * 0.15)\n",
        "test_sizeE = int(english.shape[0] * 0.15)\n",
        "test_english = english[:test_sizeE].reset_index(drop=True)\n",
        "val_english = english[test_sizeE:test_sizeE+val_sizeE].reset_index(drop=True)\n",
        "train_english = english[test_sizeE+val_sizeE:].reset_index(drop=True)"
      ],
      "metadata": {
        "id": "fb5UrzvUN89Y"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pd.concat([chinese[0:1],english[0:1],spanish[0:1],marathi[0:1],urdu[0:1]])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oS4bvFV1pZDB",
        "outputId": "bb0c39a7-5b18-44ff-ffea-bd1f3512f847"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                text label language\n",
              "0  忍不住再转一下！@LR68 @泡泡_860409 //@Tsnow-Queen: [给力][...     1  chinese\n",
              "0  its funny when you put everything you have int...     0  english\n",
              "0  Retrasamos el hasta luego España porque el avi...     0  spanish\n",
              "1    उत्तर मुंबईच्या वस्त्यां, गल्ली गल्लीमधे गोप...     0  marathi\n",
              "0             چندے سے انقلاب اور تھُک میں پکوڑے تلنا     0     urdu"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-0c297b3d-07f1-4c36-b591-49cb7597eeb2\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "      <th>language</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>忍不住再转一下！@LR68 @泡泡_860409 //@Tsnow-Queen: [给力][...</td>\n",
              "      <td>1</td>\n",
              "      <td>chinese</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>its funny when you put everything you have int...</td>\n",
              "      <td>0</td>\n",
              "      <td>english</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Retrasamos el hasta luego España porque el avi...</td>\n",
              "      <td>0</td>\n",
              "      <td>spanish</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>उत्तर मुंबईच्या वस्त्यां, गल्ली गल्लीमधे गोप...</td>\n",
              "      <td>0</td>\n",
              "      <td>marathi</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>چندے سے انقلاب اور تھُک میں پکوڑے تلنا</td>\n",
              "      <td>0</td>\n",
              "      <td>urdu</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-0c297b3d-07f1-4c36-b591-49cb7597eeb2')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-0c297b3d-07f1-4c36-b591-49cb7597eeb2 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-0c297b3d-07f1-4c36-b591-49cb7597eeb2');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Combine and Summary"
      ],
      "metadata": {
        "id": "24VySfmMmYzX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "test = pd.concat([test_english,test_chinese,test_urdu, test_spanish, test_marathi]).sample(frac=1).reset_index(drop=True)\n",
        "val = pd.concat([val_english,val_chinese,val_urdu, val_spanish, val_marathi]).sample(frac=1).reset_index(drop=True)\n",
        "train = pd.concat([train_english,train_chinese,train_urdu, train_spanish, train_marathi]).sample(frac=1).reset_index(drop=True)"
      ],
      "metadata": {
        "id": "Z9EHy_sKmdgJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "testSummary = pd.DataFrame(columns = test['language'].unique())\n",
        "temp=[0,0,0,0,0]\n",
        "testSummary.loc['positive']=temp\n",
        "testSummary.loc['negative']=temp\n",
        "testSummary.loc['total']=temp\n",
        "for lang in test['language'].unique():\n",
        "  pos=test.loc[(test.language == lang)]['label'].value_counts()[1]\n",
        "  neg=test.loc[(test.language == lang)]['label'].value_counts()[0]\n",
        "  testSummary.loc['positive'][lang]=pos\n",
        "  testSummary.loc['negative'][lang]=neg\n",
        "  testSummary.loc['total'][lang]=pos+neg\n",
        "testSummary['total'] = testSummary.sum(axis=1)\n",
        "testSummary"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 143
        },
        "id": "KpQzgbOKIBr9",
        "outputId": "ff569e21-f5f9-4c15-9380-2e69dd05c22d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "         spanish chinese english marathi urdu   total\n",
              "positive     228     659     675     605   72  2239.0\n",
              "negative     560     691     675     606   74  2606.0\n",
              "total        788    1350    1350    1211  146  4845.0"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-a453e7ea-c40b-434d-8b54-dc67c8fb8ae3\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>spanish</th>\n",
              "      <th>chinese</th>\n",
              "      <th>english</th>\n",
              "      <th>marathi</th>\n",
              "      <th>urdu</th>\n",
              "      <th>total</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>positive</th>\n",
              "      <td>228</td>\n",
              "      <td>659</td>\n",
              "      <td>675</td>\n",
              "      <td>605</td>\n",
              "      <td>72</td>\n",
              "      <td>2239.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>negative</th>\n",
              "      <td>560</td>\n",
              "      <td>691</td>\n",
              "      <td>675</td>\n",
              "      <td>606</td>\n",
              "      <td>74</td>\n",
              "      <td>2606.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>total</th>\n",
              "      <td>788</td>\n",
              "      <td>1350</td>\n",
              "      <td>1350</td>\n",
              "      <td>1211</td>\n",
              "      <td>146</td>\n",
              "      <td>4845.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-a453e7ea-c40b-434d-8b54-dc67c8fb8ae3')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-a453e7ea-c40b-434d-8b54-dc67c8fb8ae3 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-a453e7ea-c40b-434d-8b54-dc67c8fb8ae3');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 66
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "valSummary = pd.DataFrame(columns = val['language'].unique())\n",
        "temp=[0,0,0,0,0]\n",
        "valSummary.loc['positive']=temp\n",
        "valSummary.loc['negative']=temp\n",
        "valSummary.loc['total']=temp\n",
        "for lang in val['language'].unique():\n",
        "  pos=val.loc[(val.language == lang)]['label'].value_counts()[1]\n",
        "  neg=val.loc[(val.language == lang)]['label'].value_counts()[0]\n",
        "  valSummary.loc['positive'][lang]=pos\n",
        "  valSummary.loc['negative'][lang]=neg\n",
        "  valSummary.loc['total'][lang]=pos+neg\n",
        "valSummary['total'] = valSummary.sum(axis=1)\n",
        "valSummary"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 260
        },
        "id": "56qmi0k6MlPX",
        "outputId": "89e10688-57c2-4069-9ae8-2e67510db466"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "         chinese english marathi spanish urdu   total\n",
              "positive     697     699     585     241   74  2296.0\n",
              "negative     653     651     626     547   72  2549.0\n",
              "total       1350    1350    1211     788  146  4845.0"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-68b01834-28cc-4828-a2b1-006772398473\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>chinese</th>\n",
              "      <th>english</th>\n",
              "      <th>marathi</th>\n",
              "      <th>spanish</th>\n",
              "      <th>urdu</th>\n",
              "      <th>total</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>positive</th>\n",
              "      <td>697</td>\n",
              "      <td>699</td>\n",
              "      <td>585</td>\n",
              "      <td>241</td>\n",
              "      <td>74</td>\n",
              "      <td>2296.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>negative</th>\n",
              "      <td>653</td>\n",
              "      <td>651</td>\n",
              "      <td>626</td>\n",
              "      <td>547</td>\n",
              "      <td>72</td>\n",
              "      <td>2549.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>total</th>\n",
              "      <td>1350</td>\n",
              "      <td>1350</td>\n",
              "      <td>1211</td>\n",
              "      <td>788</td>\n",
              "      <td>146</td>\n",
              "      <td>4845.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-68b01834-28cc-4828-a2b1-006772398473')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-68b01834-28cc-4828-a2b1-006772398473 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-68b01834-28cc-4828-a2b1-006772398473');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 36
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Error: Runtime no longer has a reference to this dataframe, please re-run this cell and try again.\n",
            "Error: Runtime no longer has a reference to this dataframe, please re-run this cell and try again.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "trainSummary = pd.DataFrame(columns = train['language'].unique())\n",
        "temp=[0,0,0,0,0]\n",
        "trainSummary.loc['positive']=temp\n",
        "trainSummary.loc['negative']=temp\n",
        "trainSummary.loc['total']=temp\n",
        "for lang in train['language'].unique():\n",
        "  pos=train.loc[(train.language == lang)]['label'].value_counts()[1]\n",
        "  neg=train.loc[(train.language == lang)]['label'].value_counts()[0]\n",
        "  trainSummary.loc['positive'][lang]=pos\n",
        "  trainSummary.loc['negative'][lang]=neg\n",
        "  trainSummary.loc['total'][lang]=pos+neg\n",
        "trainSummary['total'] = trainSummary.sum(axis=1)\n",
        "trainSummary"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 143
        },
        "id": "Ss_3LXwuMxeJ",
        "outputId": "0a8147ea-9b96-429e-dc09-dbaa34ccb628"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "         english chinese spanish urdu marathi    total\n",
              "positive    3162    3174    1020  334    2848  10538.0\n",
              "negative    3138    3126    2662  353    2806  12085.0\n",
              "total       6300    6300    3682  687    5654  22623.0"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-0d06bf5d-bdbc-4214-be02-3afa7b9a6a4e\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>english</th>\n",
              "      <th>chinese</th>\n",
              "      <th>spanish</th>\n",
              "      <th>urdu</th>\n",
              "      <th>marathi</th>\n",
              "      <th>total</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>positive</th>\n",
              "      <td>3162</td>\n",
              "      <td>3174</td>\n",
              "      <td>1020</td>\n",
              "      <td>334</td>\n",
              "      <td>2848</td>\n",
              "      <td>10538.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>negative</th>\n",
              "      <td>3138</td>\n",
              "      <td>3126</td>\n",
              "      <td>2662</td>\n",
              "      <td>353</td>\n",
              "      <td>2806</td>\n",
              "      <td>12085.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>total</th>\n",
              "      <td>6300</td>\n",
              "      <td>6300</td>\n",
              "      <td>3682</td>\n",
              "      <td>687</td>\n",
              "      <td>5654</td>\n",
              "      <td>22623.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-0d06bf5d-bdbc-4214-be02-3afa7b9a6a4e')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-0d06bf5d-bdbc-4214-be02-3afa7b9a6a4e button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-0d06bf5d-bdbc-4214-be02-3afa7b9a6a4e');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "test_texts, test_labels = list(test.text), list(test.label)\n",
        "val_texts, val_labels = list(val.text), list(val.label)\n",
        "train_texts, train_labels = list(train.text), list(train.label)"
      ],
      "metadata": {
        "id": "NYW1GlMIPQCk"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Model Setup"
      ],
      "metadata": {
        "id": "7UtEYvWyeOTh"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Define parameters"
      ],
      "metadata": {
        "id": "ga8jf-7NfaUm"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Parameters for fine-tuning\n",
        "L_RATE = 3e-5\n",
        "MAX_LEN = 20\n",
        "NUM_EPOCHS = 3\n",
        "BATCH_SIZE = 64\n",
        "NUM_CORES = os.cpu_count()\n",
        "NUM_CORES"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8tifC63afZxs",
        "outputId": "d0d1c57d-8a48-49d0-ebfa-54776c6c3a71"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "2"
            ]
          },
          "metadata": {},
          "execution_count": 120
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model = XLMRobertaForSequenceClassification.from_pretrained(\n",
        "    'xlm-roberta-base', \n",
        "    num_labels = 2,\n",
        ")\n",
        "model.to(device)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YlBDa6Go2ZLz",
        "outputId": "6525f8f9-232f-4f53-faac-af4ff26af25f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of the model checkpoint at xlm-roberta-base were not used when initializing XLMRobertaForSequenceClassification: ['roberta.pooler.dense.weight', 'lm_head.decoder.weight', 'lm_head.layer_norm.weight', 'lm_head.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.bias', 'lm_head.bias', 'lm_head.dense.bias']\n",
            "- This IS expected if you are initializing XLMRobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing XLMRobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of XLMRobertaForSequenceClassification were not initialized from the model checkpoint at xlm-roberta-base and are newly initialized: ['classifier.out_proj.weight', 'classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "XLMRobertaForSequenceClassification(\n",
              "  (roberta): RobertaModel(\n",
              "    (embeddings): RobertaEmbeddings(\n",
              "      (word_embeddings): Embedding(250002, 768, padding_idx=1)\n",
              "      (position_embeddings): Embedding(514, 768, padding_idx=1)\n",
              "      (token_type_embeddings): Embedding(1, 768)\n",
              "      (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "      (dropout): Dropout(p=0.1, inplace=False)\n",
              "    )\n",
              "    (encoder): RobertaEncoder(\n",
              "      (layer): ModuleList(\n",
              "        (0): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (1): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (2): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (3): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (4): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (5): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (6): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (7): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (8): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (9): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (10): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (11): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "  )\n",
              "  (classifier): RobertaClassificationHead(\n",
              "    (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "    (dropout): Dropout(p=0.1, inplace=False)\n",
              "    (out_proj): Linear(in_features=768, out_features=2, bias=True)\n",
              "  )\n",
              ")"
            ]
          },
          "metadata": {},
          "execution_count": 121
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Tokenization"
      ],
      "metadata": {
        "id": "XAPa-YO9OXVM"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class CompDataset(Dataset):\n",
        "  \n",
        "    def __init__(self, df):\n",
        "        self.df_data = df\n",
        "\n",
        "    def __getitem__(self, index):\n",
        "        text = self.df_data.loc[index, 'text']\n",
        "        # tokenization\n",
        "        encoded_dict = rTokenizer.encode_plus(\n",
        "                    text,\n",
        "                    max_length = MAX_LEN,\n",
        "                    pad_to_max_length = True,\n",
        "                    return_attention_mask = True,\n",
        "                    return_tensors = 'pt',\n",
        "               )\n",
        "        padded_token_list = encoded_dict['input_ids'][0]\n",
        "        att_mask = encoded_dict['attention_mask'][0]\n",
        "        target = torch.tensor(self.df_data.loc[index, 'label'])\n",
        "        sample = (padded_token_list, att_mask, target)\n",
        "        return sample\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.df_data)"
      ],
      "metadata": {
        "id": "Z4q9vozESJPg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "test_data = CompDataset(test)\n",
        "train_data = CompDataset(train)\n",
        "val_data = CompDataset(val)"
      ],
      "metadata": {
        "id": "xYWw9sxRbuL4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(len(test_data))\n",
        "print(len(train_data))\n",
        "print(len(val_data))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Jzi19Qy6c_fQ",
        "outputId": "1f46ce47-5316-4476-803b-a54ce2dc1388"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "4845\n",
            "22623\n",
            "4845\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Baseline Results"
      ],
      "metadata": {
        "id": "XFT4QtuZ6gJ1"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def createBaseline(data, name):\n",
        "    test_data = CompDataset(data)\n",
        "    dataloader = torch.utils.data.DataLoader(test_data,\n",
        "                      batch_size=BATCH_SIZE,\n",
        "                      shuffle=True,\n",
        "                      num_workers=NUM_CORES)\n",
        "    model.eval()\n",
        "    torch.set_grad_enabled(False)\n",
        "    targets_list=[]\n",
        "    for j, batch in enumerate(dataloader):       \n",
        "        b_input_ids = batch[0].to(device)\n",
        "        b_input_mask = batch[1].to(device)\n",
        "        b_labels = batch[2].to(device)      \n",
        "        outputs = model(b_input_ids, \n",
        "                attention_mask=b_input_mask, \n",
        "                labels=b_labels)        \n",
        "        loss = outputs[0]\n",
        "        preds = outputs[1]\n",
        "        val_preds = preds.detach().cpu().numpy()\n",
        "        targets_np = b_labels.to('cpu').numpy()\n",
        "        targets_list.extend(targets_np)\n",
        "        if j == 0: \n",
        "            stacked_val_preds = val_preds\n",
        "        else:\n",
        "            stacked_val_preds = np.vstack((stacked_val_preds, val_preds))  \n",
        "    y_true = targets_list\n",
        "    y_pred = np.argmax(stacked_val_preds, axis=1)\n",
        "    acc = accuracy_score(y_true, y_pred)\n",
        "    f1 = f1_score(y_true, y_pred)\n",
        "    print(name,' test acc: ', acc)\n",
        "    print(name, ': test F1: ', f1)"
      ],
      "metadata": {
        "id": "OJQen_VBMwMT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "createBaseline(test_chinese, 'Chinese')\n",
        "createBaseline(test_english, 'English')\n",
        "createBaseline(test_urdu, 'Urdu')\n",
        "createBaseline(test_marathi, 'Marathi')\n",
        "createBaseline(test_spanish, 'Spanish')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "b9TytmQJND8U",
        "outputId": "fdcf5bde-9f01-441c-a613-7629ded9a83e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2291: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n",
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2291: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Chinese  test acc:  0.5118518518518519\n",
            "Chinese : test F1:  0.0\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2291: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n",
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2291: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "English  test acc:  0.49925925925925924\n",
            "English : test F1:  0.0\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2291: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n",
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2291: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Urdu  test acc:  0.5136986301369864\n",
            "Urdu : test F1:  0.0273972602739726\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2291: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2291: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Marathi  test acc:  0.4995871180842279\n",
            "Marathi : test F1:  0.0\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2291: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n",
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2291: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Spanish  test acc:  0.7106598984771574\n",
            "Spanish : test F1:  0.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Fine-tune"
      ],
      "metadata": {
        "id": "t1Kps-pmcmim"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import random\n",
        "import gc\n"
      ],
      "metadata": {
        "id": "cP-ivJC2h0gV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(model.parameters())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5Ict_3kjqDFw",
        "outputId": "1c1b6824-02f6-4ac0-9642-a4d47c5a36eb"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<generator object Module.parameters at 0x7fda97f9ee50>\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "optimizer = AdamW(model.parameters(),\n",
        "              lr = L_RATE, \n",
        "              eps = 1e-8 \n",
        "            )"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QjYYsGdljsU7",
        "outputId": "393f0b2e-3e4e-456c-a497-3898ae775c40"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:309: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train_data = CompDataset(train_marathi)\n",
        "val_data = CompDataset(val)"
      ],
      "metadata": {
        "id": "nLbwVmTGmeXv"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_dataloader = torch.utils.data.DataLoader(train_data,\n",
        "                    batch_size=BATCH_SIZE,\n",
        "                    shuffle=True,\n",
        "                    num_workers=NUM_CORES)\n",
        "val_dataloader = torch.utils.data.DataLoader(val_data,\n",
        "                    batch_size=BATCH_SIZE,\n",
        "                    shuffle=True,\n",
        "                    num_workers=NUM_CORES)"
      ],
      "metadata": {
        "id": "082WhkIgoFyv"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "gc.collect()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fHJ6EGatZK2y",
        "outputId": "56b481a0-40ee-48ca-a6fd-7e3ed3da1400"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "88"
            ]
          },
          "metadata": {},
          "execution_count": 103
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "seed_val = 101\n",
        "\n",
        "random.seed(seed_val)\n",
        "np.random.seed(seed_val)\n",
        "torch.manual_seed(seed_val)\n",
        "torch.cuda.manual_seed_all(seed_val)\n",
        "loss_values = []\n",
        "\n",
        "for epoch in range(0, NUM_EPOCHS):\n",
        "    \n",
        "    print(\"\")\n",
        "    print('======== Epoch {:} / {:} ========'.format(epoch + 1, NUM_EPOCHS))\n",
        "    \n",
        "    stacked_val_labels = []\n",
        "    targets_list = []\n",
        "\n",
        "    print('Training...')\n",
        "    model.train()\n",
        "    torch.set_grad_enabled(True)\n",
        "    total_train_loss = 0\n",
        "    gc.collect()\n",
        "\n",
        "    for i, batch in enumerate(train_dataloader):\n",
        "        b_input_ids = batch[0].to(device)\n",
        "        b_input_mask = batch[1].to(device)\n",
        "        b_labels = batch[2].to(device)\n",
        "        model.zero_grad()        \n",
        "        outputs = model(b_input_ids, \n",
        "                    attention_mask=b_input_mask,\n",
        "                    labels=b_labels)\n",
        "        loss = outputs[0]\n",
        "        total_train_loss = total_train_loss + loss.item()\n",
        "        optimizer.zero_grad()\n",
        "        loss.backward()\n",
        "        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
        "        xm.optimizer_step(optimizer, barrier=True)   \n",
        "        gc.collect()\n",
        "\n",
        "    print('Train loss:' ,total_train_loss)\n",
        "\n",
        "    gc.collect()\n",
        "\n",
        "    print('\\nValidation...')\n",
        "    model.eval()\n",
        "    torch.set_grad_enabled(False)\n",
        "    total_val_loss = 0\n",
        "    for j, batch in enumerate(val_dataloader):       \n",
        "        b_input_ids = batch[0].to(device)\n",
        "        b_input_mask = batch[1].to(device)\n",
        "        b_labels = batch[2].to(device)      \n",
        "        outputs = model(b_input_ids, \n",
        "                attention_mask=b_input_mask, \n",
        "                labels=b_labels)        \n",
        "        loss = outputs[0]\n",
        "        total_val_loss = total_val_loss + loss.item()\n",
        "        preds = outputs[1]\n",
        "        val_preds = preds.detach().cpu().numpy()\n",
        "        targets_np = b_labels.to('cpu').numpy()\n",
        "        targets_list.extend(targets_np)\n",
        "        if j == 0:  # first batch\n",
        "            stacked_val_preds = val_preds\n",
        "        else:\n",
        "            stacked_val_preds = np.vstack((stacked_val_preds, val_preds))  \n",
        "        gc.collect()\n",
        "    y_true = targets_list\n",
        "    y_pred = np.argmax(stacked_val_preds, axis=1)\n",
        "    val_acc = accuracy_score(y_true, y_pred)\n",
        "    f1 = f1_score(y_true, y_pred)\n",
        "    print('Val loss:' ,total_val_loss)\n",
        "    print('Val acc: ', val_acc)\n",
        "    print('Val f1:' ,f1)\n",
        "    gc.collect()\n",
        "\n",
        "    torch.save(model.state_dict(), 'model.pt')\n",
        "\n",
        "    gc.collect()\n",
        "\n",
        "        "
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SxEhLWIOcnsA",
        "outputId": "f4d7be57-444b-484c-bf94-67a5d9fd20e8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "======== Epoch 1 / 3 ========\n",
            "Training...\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2291: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n",
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2291: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Train loss: 49.59836035966873\n",
            "\n",
            "Validation...\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2291: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n",
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2291: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Val loss: 55.194731175899506\n",
            "Val acc:  0.6825593395252838\n",
            "Val f1: 0.5723025583982202\n",
            "\n",
            "======== Epoch 2 / 3 ========\n",
            "Training...\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2291: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2291: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Train loss: 39.86253261566162\n",
            "\n",
            "Validation...\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2291: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n",
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2291: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Val loss: 52.326828330755234\n",
            "Val acc:  0.7023735810113519\n",
            "Val f1: 0.7134340222575516\n",
            "\n",
            "======== Epoch 3 / 3 ========\n",
            "Training...\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2291: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n",
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2291: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Train loss: 32.779386192560196\n",
            "\n",
            "Validation...\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2291: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n",
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2291: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Val loss: 55.69235819578171\n",
            "Val acc:  0.7027863777089783\n",
            "Val f1: 0.7011207970112079\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "gc.collect()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8_KwjlaqnR1q",
        "outputId": "ad47e2d0-2d53-421d-d104-0825acd26c60"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "43"
            ]
          },
          "metadata": {},
          "execution_count": 58
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "c-l2y_RhX_gH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "uNUu6QXGTTWr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Result and Anlaysis"
      ],
      "metadata": {
        "id": "bui1NL5STTwc"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "def singleLangTest(data, name):\n",
        "    test_data = CompDataset(data)\n",
        "    dataloader = torch.utils.data.DataLoader(test_data,\n",
        "                      batch_size=BATCH_SIZE,\n",
        "                      shuffle=True,\n",
        "                      num_workers=NUM_CORES)\n",
        "    model.eval()\n",
        "    torch.set_grad_enabled(False)\n",
        "    targets_list=[]\n",
        "    for j, batch in enumerate(dataloader):       \n",
        "        b_input_ids = batch[0].to(device)\n",
        "        b_input_mask = batch[1].to(device)\n",
        "        b_labels = batch[2].to(device)      \n",
        "        outputs = model(b_input_ids, \n",
        "                attention_mask=b_input_mask, \n",
        "                labels=b_labels)        \n",
        "        loss = outputs[0]\n",
        "        preds = outputs[1]\n",
        "        val_preds = preds.detach().cpu().numpy()\n",
        "        targets_np = b_labels.to('cpu').numpy()\n",
        "        targets_list.extend(targets_np)\n",
        "        if j == 0: \n",
        "            stacked_val_preds = val_preds\n",
        "        else:\n",
        "            stacked_val_preds = np.vstack((stacked_val_preds, val_preds))  \n",
        "    y_true = targets_list\n",
        "    y_pred = np.argmax(stacked_val_preds, axis=1)\n",
        "    acc = accuracy_score(y_true, y_pred)\n",
        "    f1 = f1_score(y_true, y_pred)\n",
        "    precision = precision_score(y_true, y_pred)\n",
        "    recall = recall_score(y_true, y_pred)\n",
        "    # print(name)\n",
        "    print(name,'test acc: ', acc)\n",
        "    print(name,'test F1: ', f1)\n",
        "    print(name,'precision ', precision)\n",
        "    print(name,'recall ', recall)\n"
      ],
      "metadata": {
        "id": "GoAlyF6eTXop"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "singleLangTest(test_chinese,'chinese')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-CTDHgzdTnn5",
        "outputId": "1440de86-2f26-4b8c-a1a6-0825ff646226"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2291: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2291: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "chinese test acc:  0.6881481481481482\n",
            "chinese test F1:  0.6902133922001472\n",
            "chinese precision  0.67\n",
            "chinese recall  0.7116843702579666\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "singleLangTest(test_english,'english')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HHU0RTcRezL_",
        "outputId": "432593f0-f4ec-4f3c-9bf7-4a8c5951c0f9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2291: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n",
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2291: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "english test acc:  0.6422222222222222\n",
            "english test F1:  0.6403574087862994\n",
            "english precision  0.6437125748502994\n",
            "english recall  0.6370370370370371\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "singleLangTest(test_spanish,'spanish')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Hcj9ZZxhe6In",
        "outputId": "ca68e81c-c10d-47fe-a373-4c00adf0d93e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2291: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n",
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2291: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "spanish test acc:  0.7106598984771574\n",
            "spanish test F1:  0.6055363321799309\n",
            "spanish precision  0.5\n",
            "spanish recall  0.7675438596491229\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "singleLangTest(test_urdu,'urdu')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fDl8ZyJFe-DV",
        "outputId": "6fa77b21-1917-4c30-e3fe-bb23f60e98e1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2291: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n",
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2291: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "urdu test acc:  0.6095890410958904\n",
            "urdu test F1:  0.583941605839416\n",
            "urdu precision  0.6153846153846154\n",
            "urdu recall  0.5555555555555556\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "singleLangTest(test_marathi,'marathi')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aK5an72-fAvR",
        "outputId": "1a1239bf-8c1d-4b02-a3fe-feaa15c44526"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2291: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n",
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2291: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "marathi test acc:  0.8232865400495458\n",
            "marathi test F1:  0.8348765432098765\n",
            "marathi precision  0.7829232995658466\n",
            "marathi recall  0.8942148760330578\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "L9Nm8ujJfCuZ"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}