AraBERT_NER.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": " AraBERT-NER",
      "provenance": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/aimanyounises1/NLP_WEB/blob/master/AraBERT_NER.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "g4mZ8KYblg-g"
      },
      "source": [
        "#installing dependencies"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C4tlwQ8si_FI",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "854d2c5a-0ab4-4d70-9ff7-e8d3eb704e1b"
      },
      "source": [
        "import torch\n",
        "\n",
        "# If there's a GPU available...\n",
        "if torch.cuda.is_available():    \n",
        "\n",
        "    # Tell PyTorch to use the GPU.    \n",
        "    device = torch.device(\"cuda\")\n",
        "\n",
        "    print('There are %d GPU(s) available.' % torch.cuda.device_count())\n",
        "\n",
        "    print('We will use the GPU:', torch.cuda.get_device_name(0))\n",
        "    !nvidia-smi\n",
        "\n",
        "# If not...\n",
        "else:\n",
        "    print('No GPU available, using the CPU instead.')\n",
        "    device = torch.device(\"cpu\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "There are 1 GPU(s) available.\n",
            "We will use the GPU: Tesla P100-PCIE-16GB\n",
            "Sat Jan 30 08:48:56 2021       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 460.32.03    Driver Version: 418.67       CUDA Version: 10.1     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla P100-PCIE...  Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   37C    P0    27W / 250W |     10MiB / 16280MiB |      0%      Default |\n",
            "|                               |                      |                 ERR! |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y024z5AnlTLz",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7ec13e88-f8c9-44cc-9d53-9188ac0e4657"
      },
      "source": [
        "!pip install optuna\n",
        "!pip install seqeval\n",
        "!pip install sentencepiece\n",
        "!git clone https://github.com/huggingface/transformers\n",
        "!cd transformers && pip install .\n",
        "!pip install farasapy\n",
        "!pip install pyarabic\n",
        "!git clone https://github.com/aub-mind/arabert"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting optuna\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/59/b4/a1a80252cef3d8f5a0acdf6e678d6dc07e2e6964ee46d0453a2ae1af1ecb/optuna-2.4.0-py3-none-any.whl (282kB)\n",
            "\u001b[K     |████████████████████████████████| 286kB 8.0MB/s \n",
            "\u001b[?25hRequirement already satisfied: scipy!=1.4.0 in /usr/local/lib/python3.6/dist-packages (from optuna) (1.4.1)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from optuna) (1.19.5)\n",
            "Collecting cliff\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/0f/8f/3c74fa4b6c3db1051b495385f5302fc5d5aa0f180d40ce3e9a13c82f8c82/cliff-3.6.0-py3-none-any.whl (79kB)\n",
            "\u001b[K     |████████████████████████████████| 81kB 9.0MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from optuna) (4.41.1)\n",
            "Collecting colorlog\n",
            "  Downloading https://files.pythonhosted.org/packages/5e/39/0230290df0519d528d8d0ffdfd900150ed24e0076d13b1f19e279444aab1/colorlog-4.7.2-py2.py3-none-any.whl\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.6/dist-packages (from optuna) (20.8)\n",
            "Collecting alembic\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/cd/d2/1c6e91299280ef1a6dadbbd5e762a8b091d02e2340a9ff001b58ca80f536/alembic-1.5.3.tar.gz (1.1MB)\n",
            "\u001b[K     |████████████████████████████████| 1.1MB 10.2MB/s \n",
            "\u001b[?25hCollecting cmaes>=0.6.0\n",
            "  Downloading https://files.pythonhosted.org/packages/8d/3c/06c76ec8b54b9b1fad7f35e903fd25010fe3e0d41bd94cea5e6f12e0d651/cmaes-0.7.0-py3-none-any.whl\n",
            "Requirement already satisfied: sqlalchemy>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from optuna) (1.3.22)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from optuna) (1.0.0)\n",
            "Collecting cmd2!=0.8.3,>=0.8.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/27/54/af6e2703f064485d717cb311d3f9440cd302a823ba6d80a020b59eae166d/cmd2-1.4.0-py3-none-any.whl (133kB)\n",
            "\u001b[K     |████████████████████████████████| 133kB 35.1MB/s \n",
            "\u001b[?25hCollecting stevedore>=2.0.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d4/49/b602307aeac3df3384ff1fcd05da9c0376c622a6c48bb5325f28ab165b57/stevedore-3.3.0-py3-none-any.whl (49kB)\n",
            "\u001b[K     |████████████████████████████████| 51kB 8.5MB/s \n",
            "\u001b[?25hCollecting PrettyTable<0.8,>=0.7.2\n",
            "  Downloading https://files.pythonhosted.org/packages/ef/30/4b0746848746ed5941f052479e7c23d2b56d174b82f4fd34a25e389831f5/prettytable-0.7.2.tar.bz2\n",
            "Requirement already satisfied: pyparsing>=2.1.0 in /usr/local/lib/python3.6/dist-packages (from cliff->optuna) (2.4.7)\n",
            "Collecting pbr!=2.1.0,>=2.0.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/fb/48/69046506f6ac61c1eaa9a0d42d22d54673b69e176d30ca98e3f61513e980/pbr-5.5.1-py2.py3-none-any.whl (106kB)\n",
            "\u001b[K     |████████████████████████████████| 112kB 32.4MB/s \n",
            "\u001b[?25hRequirement already satisfied: PyYAML>=3.12 in /usr/local/lib/python3.6/dist-packages (from cliff->optuna) (3.13)\n",
            "Collecting Mako\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/5c/db/2d2d88b924aa4674a080aae83b59ea19d593250bfe5ed789947c21736785/Mako-1.1.4.tar.gz (479kB)\n",
            "\u001b[K     |████████████████████████████████| 481kB 25.1MB/s \n",
            "\u001b[?25hCollecting python-editor>=0.3\n",
            "  Downloading https://files.pythonhosted.org/packages/c6/d3/201fc3abe391bbae6606e6f1d598c15d367033332bd54352b12f35513717/python_editor-1.0.4-py3-none-any.whl\n",
            "Requirement already satisfied: python-dateutil in /usr/local/lib/python3.6/dist-packages (from alembic->optuna) (2.8.1)\n",
            "Requirement already satisfied: attrs>=16.3.0 in /usr/local/lib/python3.6/dist-packages (from cmd2!=0.8.3,>=0.8.0->cliff->optuna) (20.3.0)\n",
            "Collecting pyperclip>=1.6\n",
            "  Downloading https://files.pythonhosted.org/packages/6f/4c/0b1d507ad7e8bc31d690d04b4f475e74c2002d060f7994ce8c09612df707/pyperclip-1.8.1.tar.gz\n",
            "Requirement already satisfied: importlib-metadata>=1.6.0; python_version < \"3.8\" in /usr/local/lib/python3.6/dist-packages (from cmd2!=0.8.3,>=0.8.0->cliff->optuna) (3.4.0)\n",
            "Collecting colorama>=0.3.7\n",
            "  Downloading https://files.pythonhosted.org/packages/44/98/5b86278fbbf250d239ae0ecb724f8572af1c91f4a11edf4d36a206189440/colorama-0.4.4-py2.py3-none-any.whl\n",
            "Requirement already satisfied: wcwidth>=0.1.7 in /usr/local/lib/python3.6/dist-packages (from cmd2!=0.8.3,>=0.8.0->cliff->optuna) (0.2.5)\n",
            "Requirement already satisfied: MarkupSafe>=0.9.2 in /usr/local/lib/python3.6/dist-packages (from Mako->alembic->optuna) (1.1.1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.6/dist-packages (from python-dateutil->alembic->optuna) (1.15.0)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.6/dist-packages (from importlib-metadata>=1.6.0; python_version < \"3.8\"->cmd2!=0.8.3,>=0.8.0->cliff->optuna) (3.4.0)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.6/dist-packages (from importlib-metadata>=1.6.0; python_version < \"3.8\"->cmd2!=0.8.3,>=0.8.0->cliff->optuna) (3.7.4.3)\n",
            "Building wheels for collected packages: alembic, PrettyTable, Mako, pyperclip\n",
            "  Building wheel for alembic (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for alembic: filename=alembic-1.5.3-py2.py3-none-any.whl size=155547 sha256=3536a657b1903e5ac2b0379e74db8000f9d974172c2dc8a6a426326f4423f265\n",
            "  Stored in directory: /root/.cache/pip/wheels/7f/ed/1b/a66a0cbca75fd3e374bd5cc60c443e5675f10ef1f1f78ec31d\n",
            "  Building wheel for PrettyTable (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for PrettyTable: filename=prettytable-0.7.2-cp36-none-any.whl size=13702 sha256=ccdabed2799bd46cdf95369426dde661cf13a1bd5883fb1e85129db00cfb24c4\n",
            "  Stored in directory: /root/.cache/pip/wheels/80/34/1c/3967380d9676d162cb59513bd9dc862d0584e045a162095606\n",
            "  Building wheel for Mako (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for Mako: filename=Mako-1.1.4-py2.py3-none-any.whl size=75675 sha256=e52f325338da6f15f219a5bb61f38b2cc630b43145ba750e299c3c57422f7e54\n",
            "  Stored in directory: /root/.cache/pip/wheels/ad/10/d3/aeb26e20d19045e2a68e5d3cbb57432e11b5d9c92c99f98d47\n",
            "  Building wheel for pyperclip (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pyperclip: filename=pyperclip-1.8.1-cp36-none-any.whl size=11120 sha256=1d7599e8080df8f698483f180718e0b04ae8275dfb2e80b064d68f9159ba9acb\n",
            "  Stored in directory: /root/.cache/pip/wheels/44/10/3a/c830e9bb3db2c93274ea1f213a41fabde0d8cf3794251fad0c\n",
            "Successfully built alembic PrettyTable Mako pyperclip\n",
            "Installing collected packages: pyperclip, colorama, cmd2, pbr, stevedore, PrettyTable, cliff, colorlog, Mako, python-editor, alembic, cmaes, optuna\n",
            "  Found existing installation: prettytable 2.0.0\n",
            "    Uninstalling prettytable-2.0.0:\n",
            "      Successfully uninstalled prettytable-2.0.0\n",
            "Successfully installed Mako-1.1.4 PrettyTable-0.7.2 alembic-1.5.3 cliff-3.6.0 cmaes-0.7.0 cmd2-1.4.0 colorama-0.4.4 colorlog-4.7.2 optuna-2.4.0 pbr-5.5.1 pyperclip-1.8.1 python-editor-1.0.4 stevedore-3.3.0\n",
            "Collecting seqeval\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/9d/2d/233c79d5b4e5ab1dbf111242299153f3caddddbb691219f363ad55ce783d/seqeval-1.2.2.tar.gz (43kB)\n",
            "\u001b[K     |████████████████████████████████| 51kB 4.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.14.0 in /usr/local/lib/python3.6/dist-packages (from seqeval) (1.19.5)\n",
            "Requirement already satisfied: scikit-learn>=0.21.3 in /usr/local/lib/python3.6/dist-packages (from seqeval) (0.22.2.post1)\n",
            "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.6/dist-packages (from scikit-learn>=0.21.3->seqeval) (1.4.1)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-learn>=0.21.3->seqeval) (1.0.0)\n",
            "Building wheels for collected packages: seqeval\n",
            "  Building wheel for seqeval (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for seqeval: filename=seqeval-1.2.2-cp36-none-any.whl size=16171 sha256=1960896d1a93b543be26ee57daf7f5e4e10fe1322c41418d6dae4b9dcc9559f3\n",
            "  Stored in directory: /root/.cache/pip/wheels/52/df/1b/45d75646c37428f7e626214704a0e35bd3cfc32eda37e59e5f\n",
            "Successfully built seqeval\n",
            "Installing collected packages: seqeval\n",
            "Successfully installed seqeval-1.2.2\n",
            "Collecting sentencepiece\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/14/67/e42bd1181472c95c8cda79305df848264f2a7f62740995a46945d9797b67/sentencepiece-0.1.95-cp36-cp36m-manylinux2014_x86_64.whl (1.2MB)\n",
            "\u001b[K     |████████████████████████████████| 1.2MB 9.9MB/s \n",
            "\u001b[?25hInstalling collected packages: sentencepiece\n",
            "Successfully installed sentencepiece-0.1.95\n",
            "Cloning into 'transformers'...\n",
            "remote: Enumerating objects: 17, done.\u001b[K\n",
            "remote: Counting objects: 100% (17/17), done.\u001b[K\n",
            "remote: Compressing objects: 100% (14/14), done.\u001b[K\n",
            "remote: Total 60927 (delta 4), reused 4 (delta 1), pack-reused 60910\u001b[K\n",
            "Receiving objects: 100% (60927/60927), 45.65 MiB | 26.84 MiB/s, done.\n",
            "Resolving deltas: 100% (43034/43034), done.\n",
            "Processing /content/transformers\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "    Preparing wheel metadata ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers==4.3.0.dev0) (3.0.12)\n",
            "Collecting tokenizers==0.9.4\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/0f/1c/e789a8b12e28be5bc1ce2156cf87cb522b379be9cadc7ad8091a4cc107c4/tokenizers-0.9.4-cp36-cp36m-manylinux2010_x86_64.whl (2.9MB)\n",
            "\u001b[K     |████████████████████████████████| 2.9MB 8.9MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers==4.3.0.dev0) (2.23.0)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n",
            "\u001b[K     |████████████████████████████████| 890kB 37.9MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers==4.3.0.dev0) (4.41.1)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.6/dist-packages (from transformers==4.3.0.dev0) (3.4.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers==4.3.0.dev0) (1.19.5)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers==4.3.0.dev0) (2019.12.20)\n",
            "Requirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers==4.3.0.dev0) (0.8)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers==4.3.0.dev0) (20.8)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==4.3.0.dev0) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==4.3.0.dev0) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==4.3.0.dev0) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==4.3.0.dev0) (2020.12.5)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers==4.3.0.dev0) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers==4.3.0.dev0) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers==4.3.0.dev0) (1.0.0)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.6/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers==4.3.0.dev0) (3.7.4.3)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.6/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers==4.3.0.dev0) (3.4.0)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers==4.3.0.dev0) (2.4.7)\n",
            "Building wheels for collected packages: transformers\n",
            "  Building wheel for transformers (PEP 517) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for transformers: filename=transformers-4.3.0.dev0-cp36-none-any.whl size=1783920 sha256=241a25c2fc2b2ceb09eae499c6240a9b0d072869a23d5712f206ad4b7ed0f45d\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-evq1t9v6/wheels/23/19/dd/2561a4e47240cf6b307729d58e56f8077dd0c698f5992216cf\n",
            "Successfully built transformers\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp36-none-any.whl size=893261 sha256=cbf7ea7bf8d6679d8126e015a605b2741261c912b94d315c73dd4c071402b801\n",
            "  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n",
            "Successfully built sacremoses\n",
            "Installing collected packages: tokenizers, sacremoses, transformers\n",
            "Successfully installed sacremoses-0.0.43 tokenizers-0.9.4 transformers-4.3.0.dev0\n",
            "Collecting farasapy\n",
            "  Downloading https://files.pythonhosted.org/packages/ba/49/a1cea02059325e99bbd1114704140101893c629e67b15eaacb93711dc91e/farasapy-0.0.11-py3-none-any.whl\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from farasapy) (2.23.0)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from farasapy) (4.41.1)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->farasapy) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->farasapy) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->farasapy) (2020.12.5)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->farasapy) (3.0.4)\n",
            "Installing collected packages: farasapy\n",
            "Successfully installed farasapy-0.0.11\n",
            "Collecting pyarabic\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7b/e2/46728ec2f6fe14970de5c782346609f0636262c0941228f363710903aaa1/PyArabic-0.6.10.tar.gz (108kB)\n",
            "\u001b[K     |████████████████████████████████| 112kB 8.8MB/s \n",
            "\u001b[?25hBuilding wheels for collected packages: pyarabic\n",
            "  Building wheel for pyarabic (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pyarabic: filename=PyArabic-0.6.10-cp36-none-any.whl size=113324 sha256=f9d728c983ca8bd9a17293269403484e07268c84c6866fc8796ff56341cfe69f\n",
            "  Stored in directory: /root/.cache/pip/wheels/10/b8/f5/b7c1a50e6efb83544844f165a9b134afe7292585465e29b61d\n",
            "Successfully built pyarabic\n",
            "Installing collected packages: pyarabic\n",
            "Successfully installed pyarabic-0.6.10\n",
            "Cloning into 'arabert'...\n",
            "remote: Enumerating objects: 203, done.\u001b[K\n",
            "remote: Counting objects: 100% (203/203), done.\u001b[K\n",
            "remote: Compressing objects: 100% (148/148), done.\u001b[K\n",
            "remote: Total 417 (delta 96), reused 151 (delta 49), pack-reused 214\u001b[K\n",
            "Receiving objects: 100% (417/417), 3.82 MiB | 13.40 MiB/s, done.\n",
            "Resolving deltas: 100% (219/219), done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TVay9KamnC3I"
      },
      "source": [
        "#Creating training datasets"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hr84ozGinCFh"
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "\n",
        "from tqdm import tqdm_notebook as tqdm\n",
        "from sklearn.model_selection import train_test_split\n",
        "all_datasets= []"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8PhWP2JzrEci"
      },
      "source": [
        "class Dataset:\n",
        "    def __init__(\n",
        "        self,\n",
        "        name,\n",
        "        train,\n",
        "        test,\n",
        "        label_list,\n",
        "    ):\n",
        "        self.name = name\n",
        "        self.train = train\n",
        "        self.test = test\n",
        "        self.label_list = label_list\n",
        "\n",
        "all_datasets = []"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zM46fCcUtmMO"
      },
      "source": [
        "#ANERCorp"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dOhtydO6zKan"
      },
      "source": [
        "We are using the ANERCorp with the Camel Lab splits from https://camel.abudhabi.nyu.edu/anercorp/"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IN0zAV1HtIWt",
        "outputId": "7756b918-6af3-4995-ef2d-78feb7f39153"
      },
      "source": [
        "!unzip \"/content/drive/MyDrive/ANERcorp-CamelLabSplits.zip\""
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Archive:  /content/drive/MyDrive/ANERcorp-CamelLabSplits.zip\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qAhZUsL1t694"
      },
      "source": [
        "from collections import Counter\n",
        "import matplotlib.pyplot as plt"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_imv6ooFzWjx"
      },
      "source": [
        "`read_ANERcorp` return a list of examples\n",
        "\n",
        "each example has a list of tokens and a list of corresponding labels"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8iwEwhn8tsRQ"
      },
      "source": [
        "def read_ANERcorp(path):\n",
        "  with open(path,'r',encoding='utf-8') as f:\n",
        "    data = []\n",
        "    sentence = []\n",
        "    label = []\n",
        "    for line in f:\n",
        "      if line=='\\n':\n",
        "        if len(sentence) > 0:\n",
        "          data.append((sentence,label))\n",
        "          sentence = []\n",
        "          label = []\n",
        "        continue\n",
        "      splits = line.split()\n",
        "      sentence.append(splits[0])\n",
        "      label.append(splits[1])\n",
        "    if len(sentence) > 0:\n",
        "      data.append((sentence,label))\n",
        "  return data"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fKhTanHjtuJR"
      },
      "source": [
        "ANERCorp_path = './ANERcorp-CamelLabSplits/'\n",
        "data_train = read_ANERcorp(ANERCorp_path+'ANERCorp_CamelLab_train.txt')\n",
        "data_test = read_ANERcorp(ANERCorp_path+'ANERCorp_CamelLab_test.txt')\n",
        "\n",
        "print(Counter([ label for sentence in data_test for label in sentence[1]]))\n",
        "print(Counter([ label for sentence in data_train for label in sentence[1]]))\n",
        "\n",
        "label_list = list(Counter([ label for sentence in data_test for label in sentence[1]]).keys())\n",
        "print(label_list)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MBh0LiSjtyvC"
      },
      "source": [
        "print(\"Training Sentence Lengths: \")\n",
        "plt.hist([ len(sentence[0]) for sentence in data_train],bins=range(0,256,2))\n",
        "plt.show()\n",
        "print(sum([len(sentence[0]) > 512 for sentence in data_train]))\n",
        "\n",
        "print(\"Testing Sentence Lengths: \")\n",
        "plt.hist([ len(sentence[0]) for sentence in data_test],bins=range(0,256,2))\n",
        "plt.show()\n",
        "print(sum([len(sentence[0]) > 256 for sentence in data_test]))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2wxOpTM7zphE"
      },
      "source": [
        "256 seems a good choice for max_length"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WDx0yMuAuBN0"
      },
      "source": [
        "data_AJGT = Dataset(\"ANERCorp\", data_train, data_test, label_list)\n",
        "all_datasets.append(data_AJGT)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Bcwdslw7v0Q8"
      },
      "source": [
        "#Trainer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HUn2RB6Bvrxj"
      },
      "source": [
        "from arabert.preprocess import ArabertPreprocessor\n",
        "import numpy as np\n",
        "from seqeval.metrics import accuracy_score, f1_score, precision_score, recall_score, classification_report\n",
        "\n",
        "from transformers import AutoConfig, AutoModelForTokenClassification, AutoTokenizer\n",
        "from transformers import Trainer , TrainingArguments\n",
        "from transformers.trainer_utils import EvaluationStrategy\n",
        "from transformers.data.processors.utils import InputFeatures\n",
        "from torch.utils.data import Dataset\n",
        "from torch.utils.data import DataLoader\n",
        "from sklearn.utils import resample\n",
        "import logging\n",
        "import torch\n",
        "import optuna "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KfNKr05tv7cA"
      },
      "source": [
        "logging.basicConfig(level=logging.WARNING)\n",
        "logger = logging.getLogger(__name__)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R4SGYoB2EDJD"
      },
      "source": [
        "for x in all_datasets:\n",
        "  print(x.name)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vzeVFoz1wDYf"
      },
      "source": [
        "dataset_name = 'ANERCorp'\n",
        "model_name = 'aubmindlab/bert-base-arabertv02'\n",
        "task_name = 'tokenclassification'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_ieCOj90aw8X",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e135988d-38f2-4a98-edf0-7dbf97c68465"
      },
      "source": [
        "for d in all_datasets:\n",
        "  if d.name==dataset_name:\n",
        "    selected_dataset = d\n",
        "    print('Dataset found')\n",
        "    break"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Dataset found\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8wt-skK_zwZ7"
      },
      "source": [
        "Create, preprocess, and tokenize ANERCorp"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4YS7XI2bZTyz"
      },
      "source": [
        "class NERDataset:\n",
        "  def __init__(self, texts, tags, label_list, model_name, max_length):\n",
        "    self.texts = texts\n",
        "    self.tags = tags\n",
        "    self.label_map = {label: i for i, label in enumerate(label_list)}\n",
        "    self.preprocessor = ArabertPreprocessor(model_name.split(\"/\")[-1])    \n",
        "    self.pad_token_label_id = torch.nn.CrossEntropyLoss().ignore_index\n",
        "    # Use cross entropy ignore_index as padding label id so that only\n",
        "    # real label ids contribute to the loss later.\n",
        "    self.tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
        "    self.max_length = max_length\n",
        "\n",
        "     \n",
        "  def __len__(self):\n",
        "    return len(self.texts)\n",
        "  \n",
        "  def __getitem__(self, item):\n",
        "    textlist = self.texts[item]\n",
        "    tags = self.tags[item]\n",
        "\n",
        "    tokens = []\n",
        "    label_ids = []\n",
        "    for word, label in zip(textlist, tags):      \n",
        "      clean_word = self.preprocessor.preprocess(word)  \n",
        "      word_tokens = self.tokenizer.tokenize(clean_word)\n",
        "\n",
        "      if len(word_tokens) > 0:\n",
        "        tokens.extend(word_tokens)    \n",
        "        # Use the real label id for the first token of the word, and padding ids for the remaining tokens\n",
        "        label_ids.extend([self.label_map[label]] + [self.pad_token_label_id] * (len(word_tokens) - 1))\n",
        " \n",
        "    # Account for [CLS] and [SEP] with \"- 2\" and with \"- 3\" for RoBERTa.\n",
        "    special_tokens_count = self.tokenizer.num_special_tokens_to_add()\n",
        "    if len(tokens) > self.max_length - special_tokens_count:\n",
        "      tokens = tokens[: (self.max_length - special_tokens_count)]\n",
        "      label_ids = label_ids[: (self.max_length - special_tokens_count)]\n",
        "  \n",
        "    #Add the [SEP] token\n",
        "    tokens += [self.tokenizer.sep_token]\n",
        "    label_ids += [self.pad_token_label_id]\n",
        "    token_type_ids = [0] * len(tokens)\n",
        "\n",
        "    #Add the [CLS] TOKEN\n",
        "    tokens = [self.tokenizer.cls_token] + tokens\n",
        "    label_ids = [self.pad_token_label_id] + label_ids\n",
        "    token_type_ids = [0] + token_type_ids\n",
        "\n",
        "    input_ids = self.tokenizer.convert_tokens_to_ids(tokens)\n",
        "\n",
        "    # The mask has 1 for real tokens and 0 for padding tokens. Only real\n",
        "    # tokens are attended to.\n",
        "    attention_mask = [1] * len(input_ids)\n",
        "\n",
        "    # Zero-pad up to the sequence length.\n",
        "    padding_length = self.max_length - len(input_ids)\n",
        "\n",
        "    input_ids += [self.tokenizer.pad_token_id] * padding_length\n",
        "    attention_mask += [0] * padding_length\n",
        "    token_type_ids += [0] * padding_length\n",
        "    label_ids += [self.pad_token_label_id] * padding_length\n",
        "\n",
        "    assert len(input_ids) == self.max_length\n",
        "    assert len(attention_mask) == self.max_length\n",
        "    assert len(token_type_ids) == self.max_length\n",
        "    assert len(label_ids) == self.max_length\n",
        "\n",
        "    # if item < 5:\n",
        "    #   print(\"*** Example ***\")\n",
        "    #   print(\"tokens:\", \" \".join([str(x) for x in tokens]))\n",
        "    #   print(\"input_ids:\", \" \".join([str(x) for x in input_ids]))\n",
        "    #   print(\"attention_mask:\", \" \".join([str(x) for x in attention_mask]))\n",
        "    #   print(\"token_type_ids:\", \" \".join([str(x) for x in token_type_ids]))\n",
        "    #   print(\"label_ids:\", \" \".join([str(x) for x in label_ids]))\n",
        "    \n",
        "    return {\n",
        "        'input_ids' : torch.tensor(input_ids, dtype=torch.long),\n",
        "        'attention_mask' : torch.tensor(attention_mask, dtype=torch.long),\n",
        "        'token_type_ids' : torch.tensor(token_type_ids, dtype=torch.long),\n",
        "        'labels' : torch.tensor(label_ids, dtype=torch.long)       \n",
        "    }"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mciZOFz-amkV",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "194d88f4-4d1c-4df1-b817-e13740de5e9b"
      },
      "source": [
        "label_map = { v:index for index, v in enumerate(selected_dataset.label_list) }\n",
        "print(label_map)\n",
        "\n",
        "train_dataset = NERDataset(\n",
        "    texts=[x[0] for x in selected_dataset.train],\n",
        "    tags=[x[1] for x in selected_dataset.train],\n",
        "    label_list=selected_dataset.label_list,\n",
        "    model_name=model_name,\n",
        "    max_length=256\n",
        "    )\n",
        "\n",
        "test_dataset = NERDataset(\n",
        "    texts=[x[0] for x in selected_dataset.test],\n",
        "    tags=[x[1] for x in selected_dataset.test],\n",
        "    label_list=selected_dataset.label_list,\n",
        "    model_name=model_name,\n",
        "    max_length=256\n",
        "    )"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'B-LOC': 0, 'O': 1, 'B-PERS': 2, 'I-PERS': 3, 'B-ORG': 4, 'I-LOC': 5, 'I-ORG': 6, 'B-MISC': 7, 'I-MISC': 8}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jGJQ7E-uAZf2"
      },
      "source": [
        "Get NER Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lt7l0IxjbmNu"
      },
      "source": [
        "def model_init():\n",
        "    return AutoModelForTokenClassification.from_pretrained(model_name, return_dict=True, num_labels=len(label_map))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "11JSeo-Xwv4I"
      },
      "source": [
        "inv_label_map = {i: label for i, label in enumerate(label_list)}\n",
        "\n",
        "def align_predictions(predictions, label_ids):\n",
        "    preds = np.argmax(predictions, axis=2)\n",
        "\n",
        "    batch_size, seq_len = preds.shape\n",
        "\n",
        "    out_label_list = [[] for _ in range(batch_size)]\n",
        "    preds_list = [[] for _ in range(batch_size)]\n",
        "\n",
        "    for i in range(batch_size):\n",
        "        for j in range(seq_len):\n",
        "            if label_ids[i, j] != torch.nn.CrossEntropyLoss().ignore_index:\n",
        "                out_label_list[i].append(inv_label_map[label_ids[i][j]])\n",
        "                preds_list[i].append(inv_label_map[preds[i][j]])\n",
        "\n",
        "    return preds_list, out_label_list\n",
        "\n",
        "def compute_metrics(p):\n",
        "    preds_list, out_label_list = align_predictions(p.predictions,p.label_ids)\n",
        "    #print(classification_report(out_label_list, preds_list,digits=4))\n",
        "    return {\n",
        "        \"accuracy_score\": accuracy_score(out_label_list, preds_list),\n",
        "        \"precision\": precision_score(out_label_list, preds_list),\n",
        "        \"recall\": recall_score(out_label_list, preds_list),\n",
        "        \"f1\": f1_score(out_label_list, preds_list),\n",
        "    }"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dlpic7EW0sPR"
      },
      "source": [
        "# Doing Hyper parameter search with GRIDsearch sampler i.e. optuna will quit when all combination of values in the search space have been tested"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L3Z_kPB-dE_w"
      },
      "source": [
        "training_args = TrainingArguments(\"./train\")\n",
        "training_args.evaluate_during_training = True\n",
        "training_args.adam_epsilon = 1e-8\n",
        "training_args.fp16 = True\n",
        "training_args.per_device_train_batch_size = 4\n",
        "training_args.per_device_eval_batch_size = 16\n",
        "training_args.gradient_accumulation_steps = 8\n",
        "training_args.num_train_epochs= 6\n",
        "training_args.evaluation_strategy = EvaluationStrategy.EPOCH\n",
        "training_args.save_steps = 100000\n",
        "training_args.disable_tqdm = True\n",
        "training_args.lr_scheduler_type = 'cosine'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KeqRACPYheeb",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "bf7cff15-2567-4452-d714-d8c1ee86d395"
      },
      "source": [
        "steps_per_epoch = (len(selected_dataset.train)// (training_args.per_device_train_batch_size * training_args.gradient_accumulation_steps))\n",
        "total_steps = steps_per_epoch * training_args.num_train_epochs\n",
        "print(steps_per_epoch)\n",
        "print(total_steps)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "124\n",
            "744\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o8HOnY6odcwG",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6e750b53-4a87-4a60-d88c-fbd7e591f228"
      },
      "source": [
        "trainer = Trainer(\n",
        "    args=training_args,\n",
        "    train_dataset=train_dataset, \n",
        "    eval_dataset=test_dataset, \n",
        "    model_init=model_init,\n",
        "    compute_metrics=compute_metrics,\n",
        ")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at aubmindlab/bert-large-arabertv02 were not used when initializing BertForTokenClassification: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.weight', 'cls.predictions.decoder.bias', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias']\n",
            "- This IS expected if you are initializing BertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForTokenClassification were not initialized from the model checkpoint at aubmindlab/bert-large-arabertv02 and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lXwkOzRCdsIZ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f4807a91-defc-41f5-9522-b7106a447e7c"
      },
      "source": [
        "def my_hp_space(trial):\n",
        "    return {\n",
        "        \"learning_rate\": trial.suggest_float(\"learning_rate\", 2e-5, 7e-5, step=1e-5),\n",
        "        \"seed\": trial.suggest_categorical(\"seed\", [0, 1, 42, 666, 123, 12345]),\n",
        "        \"warmup_steps\": trial.suggest_int(\"warmup_steps\",0,total_steps*0.1,step=total_steps*0.1*0.5)\n",
        "    }\n",
        "\n",
        "search_space = {\n",
        "    \"learning_rate\":  list(np.arange(2e-5, 7e-5, 1e-5)),\n",
        "    \"seed\":  [0, 1, 42, 666, 123, 12345],\n",
        "    \"warmup_steps\": list(range(0, int((total_steps)*0.1)+1, int(total_steps*0.1*0.5)))\n",
        "}\n",
        "search_space"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'learning_rate': [2e-05,\n",
              "  3.0000000000000004e-05,\n",
              "  4.000000000000001e-05,\n",
              "  5.000000000000001e-05,\n",
              "  6.000000000000001e-05],\n",
              " 'seed': [0, 1, 42, 666, 123, 12345],\n",
              " 'warmup_steps': [0, 37, 74]}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GwcBc70Sfem3"
      },
      "source": [
        "def my_objective(metrics):\n",
        "    return metrics['eval_f1']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rfOtuWFBFtEe"
      },
      "source": [
        "name = \"NER-arabert-large-v02\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7l821sW7fdaY",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f028a805-7eed-4eb9-caf3-a3059bd64b8b"
      },
      "source": [
        "best_run = trainer.hyperparameter_search(direction=\"maximize\",\n",
        "                                         hp_space=my_hp_space,\n",
        "                                         compute_objective=my_objective,\n",
        "                                         n_trials=None,\n",
        "                                         pruner=optuna.pruners.NopPruner(),\n",
        "                                         sampler=optuna.samplers.GridSampler(search_space),\n",
        "                                         study_name=\"NER-arabert-large-v02\",\n",
        "                                         storage=\"sqlite:////content/drive/MyDrive/optuna_runs/{}.db\".format(name),\n",
        "                                         load_if_exists=False\n",
        "                                         )"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[32m[I 2021-01-06 19:52:08,426]\u001b[0m A new study created in RDB with name: NER-arabert-large-v02\u001b[0m\n",
            "Some weights of the model checkpoint at aubmindlab/bert-large-arabertv02 were not used when initializing BertForTokenClassification: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.weight', 'cls.predictions.decoder.bias', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias']\n",
            "- This IS expected if you are initializing BertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForTokenClassification were not initialized from the model checkpoint at aubmindlab/bert-large-arabertv02 and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "{'eval_loss': 0.14968341588974, 'eval_accuracy_score': 0.9664003853719241, 'eval_precision': 0.802754331408263, 'eval_recall': 0.7915024091108191, 'eval_f1': 0.7970886634318484, 'eval_runtime': 12.5556, 'eval_samples_per_second': 73.673, 'epoch': 0.9979879275653923}\n",
            "{'eval_loss': 0.14608950912952423, 'eval_accuracy_score': 0.9681666733571515, 'eval_precision': 0.8205709107385591, 'eval_recall': 0.7932544897065265, 'eval_f1': 0.8066815144766147, 'eval_runtime': 12.4532, 'eval_samples_per_second': 74.278, 'epoch': 1.9979879275653922}\n",
            "{'eval_loss': 0.1497354358434677, 'eval_accuracy_score': 0.9700533900686443, 'eval_precision': 0.8355173971983733, 'eval_recall': 0.8098992553657468, 'eval_f1': 0.822508896797153, 'eval_runtime': 12.4806, 'eval_samples_per_second': 74.115, 'epoch': 2.9979879275653922}\n",
            "{'eval_loss': 0.15973614156246185, 'eval_accuracy_score': 0.970213961703665, 'eval_precision': 0.835820895522388, 'eval_recall': 0.80946123521682, 'eval_f1': 0.822429906542056, 'eval_runtime': 12.427, 'eval_samples_per_second': 74.434, 'epoch': 3.9979879275653922}\n",
            "{'loss': 0.06381373596191406, 'learning_rate': 1.4563337999141146e-05, 'epoch': 4.0321931589537225}\n",
            "{'eval_loss': 0.16590078175067902, 'eval_accuracy_score': 0.970454819156196, 'eval_precision': 0.8337850045167118, 'eval_recall': 0.8085851949189663, 'eval_f1': 0.8209917722926395, 'eval_runtime': 12.4981, 'eval_samples_per_second': 74.011, 'epoch': 4.997987927565393}\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[32m[I 2021-01-06 20:05:49,234]\u001b[0m Trial 0 finished with value: 0.82018226272505 and parameters: {'learning_rate': 6.000000000000001e-05, 'seed': 123, 'warmup_steps': 0}. Best is trial 0 with value: 0.82018226272505.\u001b[0m\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "{'eval_loss': 0.16720832884311676, 'eval_accuracy_score': 0.9703343904299305, 'eval_precision': 0.8325812274368231, 'eval_recall': 0.8081471747700394, 'eval_f1': 0.82018226272505, 'eval_runtime': 12.3937, 'eval_samples_per_second': 74.635, 'epoch': 5.997987927565393}\n",
            "{'train_runtime': 795.8278, 'train_samples_per_second': 0.935, 'epoch': 5.997987927565393}\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at aubmindlab/bert-large-arabertv02 were not used when initializing BertForTokenClassification: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.weight', 'cls.predictions.decoder.bias', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias']\n",
            "- This IS expected if you are initializing BertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForTokenClassification were not initialized from the model checkpoint at aubmindlab/bert-large-arabertv02 and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "{'eval_loss': 0.16222655773162842, 'eval_accuracy_score': 0.9620649512263658, 'eval_precision': 0.8064965197215778, 'eval_recall': 0.7612790188348664, 'eval_f1': 0.783235691753042, 'eval_runtime': 12.5336, 'eval_samples_per_second': 73.802, 'epoch': 0.9979879275653923}\n",
            "{'eval_loss': 0.14534486830234528, 'eval_accuracy_score': 0.9666412428244551, 'eval_precision': 0.8104371097234612, 'eval_recall': 0.7958826106000876, 'eval_f1': 0.8030939226519337, 'eval_runtime': 12.461, 'eval_samples_per_second': 74.232, 'epoch': 1.9979879275653922}\n",
            "{'eval_loss': 0.14148828387260437, 'eval_accuracy_score': 0.9684075308096824, 'eval_precision': 0.8211564320932317, 'eval_recall': 0.8024529128339903, 'eval_f1': 0.8116969428444839, 'eval_runtime': 12.4885, 'eval_samples_per_second': 74.068, 'epoch': 2.9979879275653922}\n",
            "{'eval_loss': 0.1439645141363144, 'eval_accuracy_score': 0.96900967444101, 'eval_precision': 0.8254822790489008, 'eval_recall': 0.8059570740254052, 'eval_f1': 0.8156028368794326, 'eval_runtime': 12.4113, 'eval_samples_per_second': 74.529, 'epoch': 3.9979879275653922}\n",
            "{'loss': 0.10797702026367187, 'learning_rate': 4.854445999713715e-06, 'epoch': 4.0321931589537225}\n",
            "{'eval_loss': 0.1450127512216568, 'eval_accuracy_score': 0.9695716751635823, 'eval_precision': 0.8243423985733392, 'eval_recall': 0.8098992553657468, 'eval_f1': 0.8170570039770216, 'eval_runtime': 12.5371, 'eval_samples_per_second': 73.781, 'epoch': 4.997987927565393}\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[32m[I 2021-01-06 20:19:24,231]\u001b[0m Trial 1 finished with value: 0.8176600441501104 and parameters: {'learning_rate': 2e-05, 'seed': 42, 'warmup_steps': 0}. Best is trial 0 with value: 0.82018226272505.\u001b[0m\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "{'eval_loss': 0.1451595425605774, 'eval_accuracy_score': 0.9696118180723375, 'eval_precision': 0.8242100578549176, 'eval_recall': 0.8112133158125274, 'eval_f1': 0.8176600441501104, 'eval_runtime': 12.4537, 'eval_samples_per_second': 74.275, 'epoch': 5.997987927565393}\n",
            "{'train_runtime': 800.0686, 'train_samples_per_second': 0.93, 'epoch': 5.997987927565393}\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at aubmindlab/bert-large-arabertv02 were not used when initializing BertForTokenClassification: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.weight', 'cls.predictions.decoder.bias', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias']\n",
            "- This IS expected if you are initializing BertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForTokenClassification were not initialized from the model checkpoint at aubmindlab/bert-large-arabertv02 and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "{'eval_loss': 0.16658826172351837, 'eval_accuracy_score': 0.9608205210549556, 'eval_precision': 0.7775218793182865, 'eval_recall': 0.7393780113885239, 'eval_f1': 0.7579703637180062, 'eval_runtime': 12.3929, 'eval_samples_per_second': 74.639, 'epoch': 0.9979879275653923}\n",
            "{'eval_loss': 0.15009905397891998, 'eval_accuracy_score': 0.9678455300871102, 'eval_precision': 0.8320074871314928, 'eval_recall': 0.7787998247919404, 'eval_f1': 0.804524886877828, 'eval_runtime': 12.3873, 'eval_samples_per_second': 74.674, 'epoch': 1.9979879275653922}\n",
            "{'eval_loss': 0.13599847257137299, 'eval_accuracy_score': 0.9703343904299305, 'eval_precision': 0.8415300546448088, 'eval_recall': 0.80946123521682, 'eval_f1': 0.8251841929002011, 'eval_runtime': 12.5291, 'eval_samples_per_second': 73.828, 'epoch': 2.9979879275653922}\n",
            "{'eval_loss': 0.14517879486083984, 'eval_accuracy_score': 0.9704949620649512, 'eval_precision': 0.8349864743011722, 'eval_recall': 0.8112133158125274, 'eval_f1': 0.8229282381692958, 'eval_runtime': 12.4052, 'eval_samples_per_second': 74.566, 'epoch': 3.9979879275653922}\n",
            "{'loss': 0.14246501159667968, 'learning_rate': 1.7584097307467283e-05, 'epoch': 4.0321931589537225}\n",
            "{'eval_loss': 0.1591440588235855, 'eval_accuracy_score': 0.9696519609810927, 'eval_precision': 0.8367533059735522, 'eval_recall': 0.803766973280771, 'eval_f1': 0.819928507596068, 'eval_runtime': 12.2991, 'eval_samples_per_second': 75.209, 'epoch': 4.997987927565393}\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[32m[I 2021-01-06 20:32:53,286]\u001b[0m Trial 2 finished with value: 0.8212689901697944 and parameters: {'learning_rate': 6.000000000000001e-05, 'seed': 123, 'warmup_steps': 74}. Best is trial 2 with value: 0.8212689901697944.\u001b[0m\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "{'eval_loss': 0.159962460398674, 'eval_accuracy_score': 0.9699329613423788, 'eval_precision': 0.8381212950296397, 'eval_recall': 0.8050810337275515, 'eval_f1': 0.8212689901697944, 'eval_runtime': 12.2315, 'eval_samples_per_second': 75.624, 'epoch': 5.997987927565393}\n",
            "{'train_runtime': 794.7682, 'train_samples_per_second': 0.936, 'epoch': 5.997987927565393}\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at aubmindlab/bert-large-arabertv02 were not used when initializing BertForTokenClassification: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.weight', 'cls.predictions.decoder.bias', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias']\n",
            "- This IS expected if you are initializing BertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForTokenClassification were not initialized from the model checkpoint at aubmindlab/bert-large-arabertv02 and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "{'eval_loss': 0.1652948558330536, 'eval_accuracy_score': 0.9615029505037935, 'eval_precision': 0.7961520842876775, 'eval_recall': 0.7612790188348664, 'eval_f1': 0.7783251231527093, 'eval_runtime': 12.2261, 'eval_samples_per_second': 75.658, 'epoch': 0.9979879275653923}\n",
            "{'eval_loss': 0.14061038196086884, 'eval_accuracy_score': 0.9675243868170688, 'eval_precision': 0.8112788632326821, 'eval_recall': 0.8002628120893561, 'eval_f1': 0.8057331863285557, 'eval_runtime': 12.2606, 'eval_samples_per_second': 75.445, 'epoch': 1.9979879275653922}\n",
            "{'eval_loss': 0.13811887800693512, 'eval_accuracy_score': 0.9689293886234996, 'eval_precision': 0.8247882300490414, 'eval_recall': 0.8103372755146737, 'eval_f1': 0.8174988952717631, 'eval_runtime': 12.3675, 'eval_samples_per_second': 74.793, 'epoch': 2.9979879275653922}\n",
            "{'eval_loss': 0.14436091482639313, 'eval_accuracy_score': 0.9691702460760306, 'eval_precision': 0.8278798744957419, 'eval_recall': 0.8090232150678931, 'eval_f1': 0.8183429330970314, 'eval_runtime': 12.5351, 'eval_samples_per_second': 73.793, 'epoch': 3.9979879275653922}\n",
            "{'loss': 0.16509564208984376, 'learning_rate': 5.3240401923161935e-06, 'epoch': 4.0321931589537225}\n",
            "{'eval_loss': 0.14579527080059052, 'eval_accuracy_score': 0.9698526755248685, 'eval_precision': 0.8298347476552033, 'eval_recall': 0.8138414367060884, 'eval_f1': 0.8217602830605926, 'eval_runtime': 12.5003, 'eval_samples_per_second': 73.998, 'epoch': 4.997987927565393}\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[32m[I 2021-01-06 20:46:21,043]\u001b[0m Trial 3 finished with value: 0.8230088495575222 and parameters: {'learning_rate': 2e-05, 'seed': 42, 'warmup_steps': 37}. Best is trial 3 with value: 0.8230088495575222.\u001b[0m\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "{'eval_loss': 0.14580576121807098, 'eval_accuracy_score': 0.9700533900686443, 'eval_precision': 0.8314707197139025, 'eval_recall': 0.8147174770039421, 'eval_f1': 0.8230088495575222, 'eval_runtime': 12.3025, 'eval_samples_per_second': 75.188, 'epoch': 5.997987927565393}\n",
            "{'train_runtime': 793.6147, 'train_samples_per_second': 0.937, 'epoch': 5.997987927565393}\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at aubmindlab/bert-large-arabertv02 were not used when initializing BertForTokenClassification: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.weight', 'cls.predictions.decoder.bias', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias']\n",
            "- This IS expected if you are initializing BertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForTokenClassification were not initialized from the model checkpoint at aubmindlab/bert-large-arabertv02 and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "{'eval_loss': 0.14605310559272766, 'eval_accuracy_score': 0.9674039580908033, 'eval_precision': 0.8357109986194201, 'eval_recall': 0.7954445904511608, 'eval_f1': 0.8150807899461401, 'eval_runtime': 12.5349, 'eval_samples_per_second': 73.794, 'epoch': 0.9979879275653923}\n",
            "{'eval_loss': 0.13993820548057556, 'eval_accuracy_score': 0.9701336758861547, 'eval_precision': 0.822400713966979, 'eval_recall': 0.8072711344721857, 'eval_f1': 0.8147656940760388, 'eval_runtime': 12.3697, 'eval_samples_per_second': 74.78, 'epoch': 1.9979879275653922}\n",
            "{'eval_loss': 0.14386527240276337, 'eval_accuracy_score': 0.9703745333386857, 'eval_precision': 0.8413699864803966, 'eval_recall': 0.8177836180464302, 'eval_f1': 0.8294091514882276, 'eval_runtime': 12.5571, 'eval_samples_per_second': 73.664, 'epoch': 2.9979879275653922}\n",
            "{'eval_loss': 0.15302197635173798, 'eval_accuracy_score': 0.9710569627875236, 'eval_precision': 0.8432700993676604, 'eval_recall': 0.8177836180464302, 'eval_f1': 0.8303313319991106, 'eval_runtime': 12.4268, 'eval_samples_per_second': 74.436, 'epoch': 3.9979879275653922}\n",
            "{'loss': 0.10708082580566407, 'learning_rate': 1.597212057694858e-05, 'epoch': 4.0321931589537225}\n",
            "{'eval_loss': 0.15431758761405945, 'eval_accuracy_score': 0.9714583918750753, 'eval_precision': 0.8461191335740073, 'eval_recall': 0.8212877792378449, 'eval_f1': 0.8335185596799289, 'eval_runtime': 12.5264, 'eval_samples_per_second': 73.844, 'epoch': 4.997987927565393}\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[32m[I 2021-01-06 20:59:55,314]\u001b[0m Trial 4 finished with value: 0.8322981366459629 and parameters: {'learning_rate': 6.000000000000001e-05, 'seed': 666, 'warmup_steps': 37}. Best is trial 4 with value: 0.8322981366459629.\u001b[0m\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "{'eval_loss': 0.1557389199733734, 'eval_accuracy_score': 0.9714182489663201, 'eval_precision': 0.8431460674157303, 'eval_recall': 0.8217257993867718, 'eval_f1': 0.8322981366459629, 'eval_runtime': 12.4299, 'eval_samples_per_second': 74.417, 'epoch': 5.997987927565393}\n",
            "{'train_runtime': 799.1988, 'train_samples_per_second': 0.931, 'epoch': 5.997987927565393}\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at aubmindlab/bert-large-arabertv02 were not used when initializing BertForTokenClassification: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.weight', 'cls.predictions.decoder.bias', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias']\n",
            "- This IS expected if you are initializing BertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForTokenClassification were not initialized from the model checkpoint at aubmindlab/bert-large-arabertv02 and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "{'eval_loss': 0.14869055151939392, 'eval_accuracy_score': 0.9663200995544137, 'eval_precision': 0.8041282389108476, 'eval_recall': 0.8020148926850635, 'eval_f1': 0.8030701754385966, 'eval_runtime': 12.3688, 'eval_samples_per_second': 74.785, 'epoch': 0.9979879275653923}\n",
            "{'eval_loss': 0.13985051214694977, 'eval_accuracy_score': 0.96900967444101, 'eval_precision': 0.8352835283528353, 'eval_recall': 0.8129653964082347, 'eval_f1': 0.8239733629300776, 'eval_runtime': 12.5626, 'eval_samples_per_second': 73.631, 'epoch': 1.9979879275653922}\n",
            "{'eval_loss': 0.14176931977272034, 'eval_accuracy_score': 0.9709766769700132, 'eval_precision': 0.8369905956112853, 'eval_recall': 0.8186596583442839, 'eval_f1': 0.8277236492471214, 'eval_runtime': 12.3058, 'eval_samples_per_second': 75.168, 'epoch': 2.9979879275653922}\n",
            "{'eval_loss': 0.1505139321088791, 'eval_accuracy_score': 0.9708161053349925, 'eval_precision': 0.8388838883888389, 'eval_recall': 0.8164695575996496, 'eval_f1': 0.8275249722530521, 'eval_runtime': 12.4079, 'eval_samples_per_second': 74.549, 'epoch': 3.9979879275653922}\n",
            "{'loss': 0.07361382293701171, 'learning_rate': 1.213611499928429e-05, 'epoch': 4.0321931589537225}\n",
            "{'eval_loss': 0.15390272438526154, 'eval_accuracy_score': 0.9710168198787684, 'eval_precision': 0.8436232537178909, 'eval_recall': 0.8199737187910644, 'eval_f1': 0.8316303864948912, 'eval_runtime': 12.3189, 'eval_samples_per_second': 75.088, 'epoch': 4.997987927565393}\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[32m[I 2021-01-06 21:13:24,958]\u001b[0m Trial 5 finished with value: 0.8326693227091634 and parameters: {'learning_rate': 5.000000000000001e-05, 'seed': 42, 'warmup_steps': 0}. Best is trial 5 with value: 0.8326693227091634.\u001b[0m\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "{'eval_loss': 0.15310879051685333, 'eval_accuracy_score': 0.9712576773312994, 'eval_precision': 0.8416107382550335, 'eval_recall': 0.823915900131406, 'eval_f1': 0.8326693227091634, 'eval_runtime': 12.4731, 'eval_samples_per_second': 74.16, 'epoch': 5.997987927565393}\n",
            "{'train_runtime': 795.4501, 'train_samples_per_second': 0.935, 'epoch': 5.997987927565393}\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at aubmindlab/bert-large-arabertv02 were not used when initializing BertForTokenClassification: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.weight', 'cls.predictions.decoder.bias', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias']\n",
            "- This IS expected if you are initializing BertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForTokenClassification were not initialized from the model checkpoint at aubmindlab/bert-large-arabertv02 and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "{'eval_loss': 0.15217162668704987, 'eval_accuracy_score': 0.9663200995544137, 'eval_precision': 0.794351279788173, 'eval_recall': 0.7884362680683311, 'eval_f1': 0.7913827214772478, 'eval_runtime': 12.4719, 'eval_samples_per_second': 74.167, 'epoch': 0.9979879275653923}\n",
            "{'eval_loss': 0.14081041514873505, 'eval_accuracy_score': 0.9701336758861547, 'eval_precision': 0.822594880847308, 'eval_recall': 0.8164695575996496, 'eval_f1': 0.8195207737964387, 'eval_runtime': 12.4517, 'eval_samples_per_second': 74.287, 'epoch': 1.9979879275653922}\n",
            "{'eval_loss': 0.15071506798267365, 'eval_accuracy_score': 0.9696519609810927, 'eval_precision': 0.8277877697841727, 'eval_recall': 0.806395094174332, 'eval_f1': 0.8169514089194586, 'eval_runtime': 12.4466, 'eval_samples_per_second': 74.318, 'epoch': 2.9979879275653922}\n",
            "{'eval_loss': 0.15828166902065277, 'eval_accuracy_score': 0.969732246798603, 'eval_precision': 0.823943661971831, 'eval_recall': 0.8199737187910644, 'eval_f1': 0.821953896816685, 'eval_runtime': 12.5697, 'eval_samples_per_second': 73.59, 'epoch': 3.9979879275653922}\n",
            "{'loss': 0.0856999969482422, 'learning_rate': 1.4563337999141146e-05, 'epoch': 4.0321931589537225}\n",
            "{'eval_loss': 0.1637023240327835, 'eval_accuracy_score': 0.9701738187949098, 'eval_precision': 0.828028293545535, 'eval_recall': 0.8204117389399912, 'eval_f1': 0.8242024202420242, 'eval_runtime': 12.4763, 'eval_samples_per_second': 74.141, 'epoch': 4.997987927565393}\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[32m[I 2021-01-06 21:27:14,968]\u001b[0m Trial 6 finished with value: 0.8251101321585902 and parameters: {'learning_rate': 6.000000000000001e-05, 'seed': 12345, 'warmup_steps': 0}. Best is trial 5 with value: 0.8326693227091634.\u001b[0m\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "{'eval_loss': 0.1653466671705246, 'eval_accuracy_score': 0.9703745333386857, 'eval_precision': 0.8298626495347807, 'eval_recall': 0.8204117389399912, 'eval_f1': 0.8251101321585902, 'eval_runtime': 13.4926, 'eval_samples_per_second': 68.556, 'epoch': 5.997987927565393}\n",
            "{'train_runtime': 801.1079, 'train_samples_per_second': 0.929, 'epoch': 5.997987927565393}\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at aubmindlab/bert-large-arabertv02 were not used when initializing BertForTokenClassification: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.weight', 'cls.predictions.decoder.bias', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias']\n",
            "- This IS expected if you are initializing BertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForTokenClassification were not initialized from the model checkpoint at aubmindlab/bert-large-arabertv02 and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "{'eval_loss': 0.16469930112361908, 'eval_accuracy_score': 0.9651158122917587, 'eval_precision': 0.8257611241217798, 'eval_recall': 0.7722295225580377, 'eval_f1': 0.7980986871887732, 'eval_runtime': 12.4247, 'eval_samples_per_second': 74.449, 'epoch': 0.9979879275653923}\n",
            "{'eval_loss': 0.13692522048950195, 'eval_accuracy_score': 0.9698928184336237, 'eval_precision': 0.8266309204647007, 'eval_recall': 0.8103372755146737, 'eval_f1': 0.8184030081840301, 'eval_runtime': 12.4685, 'eval_samples_per_second': 74.187, 'epoch': 1.9979879275653922}\n",
            "{'eval_loss': 0.14905139803886414, 'eval_accuracy_score': 0.970695676608727, 'eval_precision': 0.8394226432115471, 'eval_recall': 0.8151554971528691, 'eval_f1': 0.8271111111111111, 'eval_runtime': 12.5429, 'eval_samples_per_second': 73.747, 'epoch': 2.9979879275653922}\n",
            "{'eval_loss': 0.1504199504852295, 'eval_accuracy_score': 0.9715788206013408, 'eval_precision': 0.8405211141060198, 'eval_recall': 0.8195356986421375, 'eval_f1': 0.8298957640275005, 'eval_runtime': 12.5593, 'eval_samples_per_second': 73.651, 'epoch': 3.9979879275653922}\n",
            "{'loss': 0.10845223236083984, 'learning_rate': 1.3310100480790485e-05, 'epoch': 4.0321931589537225}\n",
            "{'eval_loss': 0.1586398035287857, 'eval_accuracy_score': 0.9714985347838304, 'eval_precision': 0.8404112650871703, 'eval_recall': 0.8234778799824792, 'eval_f1': 0.8318584070796461, 'eval_runtime': 12.5732, 'eval_samples_per_second': 73.569, 'epoch': 4.997987927565393}\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[32m[I 2021-01-06 21:41:04,163]\u001b[0m Trial 7 finished with value: 0.832110295752724 and parameters: {'learning_rate': 5.000000000000001e-05, 'seed': 0, 'warmup_steps': 37}. Best is trial 5 with value: 0.8326693227091634.\u001b[0m\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "{'eval_loss': 0.16083067655563354, 'eval_accuracy_score': 0.9712576773312994, 'eval_precision': 0.8450767841011744, 'eval_recall': 0.8195356986421375, 'eval_f1': 0.832110295752724, 'eval_runtime': 12.4706, 'eval_samples_per_second': 74.174, 'epoch': 5.997987927565393}\n",
            "{'train_runtime': 800.6381, 'train_samples_per_second': 0.929, 'epoch': 5.997987927565393}\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at aubmindlab/bert-large-arabertv02 were not used when initializing BertForTokenClassification: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.weight', 'cls.predictions.decoder.bias', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias']\n",
            "- This IS expected if you are initializing BertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForTokenClassification were not initialized from the model checkpoint at aubmindlab/bert-large-arabertv02 and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "{'eval_loss': 0.14369803667068481, 'eval_accuracy_score': 0.9675243868170688, 'eval_precision': 0.8102222222222222, 'eval_recall': 0.7985107314936487, 'eval_f1': 0.8043238473417162, 'eval_runtime': 12.5132, 'eval_samples_per_second': 73.922, 'epoch': 0.9979879275653923}\n",
            "{'eval_loss': 0.14006587862968445, 'eval_accuracy_score': 0.9694512464373168, 'eval_precision': 0.8135593220338984, 'eval_recall': 0.8199737187910644, 'eval_f1': 0.8167539267015707, 'eval_runtime': 12.5392, 'eval_samples_per_second': 73.769, 'epoch': 1.9979879275653922}\n",
            "{'eval_loss': 0.13987639546394348, 'eval_accuracy_score': 0.970213961703665, 'eval_precision': 0.8281389136242209, 'eval_recall': 0.8147174770039421, 'eval_f1': 0.821373371605211, 'eval_runtime': 12.5195, 'eval_samples_per_second': 73.885, 'epoch': 2.9979879275653922}\n",
            "{'eval_loss': 0.15173552930355072, 'eval_accuracy_score': 0.9706555336999719, 'eval_precision': 0.8300884955752212, 'eval_recall': 0.8217257993867718, 'eval_f1': 0.8258859784283514, 'eval_runtime': 12.5091, 'eval_samples_per_second': 73.946, 'epoch': 3.9979879275653922}\n",
            "{'loss': 0.07788986206054688, 'learning_rate': 9.708891999427432e-06, 'epoch': 4.0321931589537225}\n",
            "{'eval_loss': 0.1567077934741974, 'eval_accuracy_score': 0.9704146762474409, 'eval_precision': 0.8249889233495791, 'eval_recall': 0.8155935173017959, 'eval_f1': 0.8202643171806167, 'eval_runtime': 12.4142, 'eval_samples_per_second': 74.511, 'epoch': 4.997987927565393}\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[32m[I 2021-01-06 21:54:36,897]\u001b[0m Trial 8 finished with value: 0.8214522180534098 and parameters: {'learning_rate': 4.000000000000001e-05, 'seed': 12345, 'warmup_steps': 0}. Best is trial 5 with value: 0.8326693227091634.\u001b[0m\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "{'eval_loss': 0.15830431878566742, 'eval_accuracy_score': 0.9705752478824615, 'eval_precision': 0.827846975088968, 'eval_recall': 0.8151554971528691, 'eval_f1': 0.8214522180534098, 'eval_runtime': 12.3759, 'eval_samples_per_second': 74.742, 'epoch': 5.997987927565393}\n",
            "{'train_runtime': 798.0757, 'train_samples_per_second': 0.932, 'epoch': 5.997987927565393}\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at aubmindlab/bert-large-arabertv02 were not used when initializing BertForTokenClassification: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.weight', 'cls.predictions.decoder.bias', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias']\n",
            "- This IS expected if you are initializing BertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForTokenClassification were not initialized from the model checkpoint at aubmindlab/bert-large-arabertv02 and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "{'eval_loss': 0.13126732409000397, 'eval_accuracy_score': 0.9705752478824615, 'eval_precision': 0.8194993412384717, 'eval_recall': 0.8173455978975033, 'eval_f1': 0.8184210526315789, 'eval_runtime': 12.5629, 'eval_samples_per_second': 73.63, 'epoch': 0.9979879275653923}\n",
            "{'eval_loss': 0.19756591320037842, 'eval_accuracy_score': 0.9571273734494802, 'eval_precision': 0.7854406130268199, 'eval_recall': 0.718353044240035, 'eval_f1': 0.750400366048959, 'eval_runtime': 12.5069, 'eval_samples_per_second': 73.959, 'epoch': 1.9979879275653922}\n",
            "{'eval_loss': 0.14620041847229004, 'eval_accuracy_score': 0.9686885311709686, 'eval_precision': 0.8307830783078308, 'eval_recall': 0.8085851949189663, 'eval_f1': 0.8195338512763597, 'eval_runtime': 12.4833, 'eval_samples_per_second': 74.099, 'epoch': 2.9979879275653922}\n",
            "{'eval_loss': 0.14146599173545837, 'eval_accuracy_score': 0.9710168198787684, 'eval_precision': 0.833185053380783, 'eval_recall': 0.8204117389399912, 'eval_f1': 0.8267490620172148, 'eval_runtime': 12.5503, 'eval_samples_per_second': 73.704, 'epoch': 3.9979879275653922}\n",
            "{'loss': 0.09517868041992188, 'learning_rate': 1.213611499928429e-05, 'epoch': 4.0321931589537225}\n",
            "{'eval_loss': 0.14537017047405243, 'eval_accuracy_score': 0.9711773915137891, 'eval_precision': 0.8363880196691998, 'eval_recall': 0.8195356986421375, 'eval_f1': 0.8278761061946903, 'eval_runtime': 12.5102, 'eval_samples_per_second': 73.94, 'epoch': 4.997987927565393}\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[32m[I 2021-01-06 22:08:05,414]\u001b[0m Trial 9 finished with value: 0.8285777482857773 and parameters: {'learning_rate': 5.000000000000001e-05, 'seed': 1, 'warmup_steps': 0}. Best is trial 5 with value: 0.8326693227091634.\u001b[0m\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "{'eval_loss': 0.14718475937843323, 'eval_accuracy_score': 0.9711372486050339, 'eval_precision': 0.8369079535299374, 'eval_recall': 0.8204117389399912, 'eval_f1': 0.8285777482857773, 'eval_runtime': 12.5179, 'eval_samples_per_second': 73.894, 'epoch': 5.997987927565393}\n",
            "{'train_runtime': 793.9273, 'train_samples_per_second': 0.937, 'epoch': 5.997987927565393}\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at aubmindlab/bert-large-arabertv02 were not used when initializing BertForTokenClassification: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.weight', 'cls.predictions.decoder.bias', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias']\n",
            "- This IS expected if you are initializing BertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForTokenClassification were not initialized from the model checkpoint at aubmindlab/bert-large-arabertv02 and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "{'eval_loss': 0.1570618599653244, 'eval_accuracy_score': 0.9633896672152864, 'eval_precision': 0.8187762727697337, 'eval_recall': 0.7678493210687691, 'eval_f1': 0.7924954792043399, 'eval_runtime': 12.4615, 'eval_samples_per_second': 74.229, 'epoch': 0.9979879275653923}\n",
            "{'eval_loss': 0.1403229534626007, 'eval_accuracy_score': 0.9693709606198065, 'eval_precision': 0.8144375553587245, 'eval_recall': 0.8055190538764783, 'eval_f1': 0.8099537546795859, 'eval_runtime': 12.5189, 'eval_samples_per_second': 73.889, 'epoch': 1.9979879275653922}\n",
            "{'eval_loss': 0.14377336204051971, 'eval_accuracy_score': 0.9696118180723375, 'eval_precision': 0.8271549799017418, 'eval_recall': 0.8112133158125274, 'eval_f1': 0.8191065900044228, 'eval_runtime': 12.4886, 'eval_samples_per_second': 74.067, 'epoch': 2.9979879275653922}\n",
            "{'eval_loss': 0.14475049078464508, 'eval_accuracy_score': 0.9709766769700132, 'eval_precision': 0.8302895322939866, 'eval_recall': 0.8164695575996496, 'eval_f1': 0.823321554770318, 'eval_runtime': 12.575, 'eval_samples_per_second': 73.559, 'epoch': 3.9979879275653922}\n",
            "{'loss': 0.1311641845703125, 'learning_rate': 7.98606028847429e-06, 'epoch': 4.0321931589537225}\n",
            "{'eval_loss': 0.14955629408359528, 'eval_accuracy_score': 0.9704949620649512, 'eval_precision': 0.8356471115091805, 'eval_recall': 0.8173455978975033, 'eval_f1': 0.8263950398582816, 'eval_runtime': 12.5209, 'eval_samples_per_second': 73.876, 'epoch': 4.997987927565393}\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[32m[I 2021-01-06 22:21:39,242]\u001b[0m Trial 10 finished with value: 0.8267140004437542 and parameters: {'learning_rate': 3.0000000000000004e-05, 'seed': 0, 'warmup_steps': 37}. Best is trial 5 with value: 0.8326693227091634.\u001b[0m\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "{'eval_loss': 0.15130630135536194, 'eval_accuracy_score': 0.9703343904299305, 'eval_precision': 0.8376798561151079, 'eval_recall': 0.8160315374507228, 'eval_f1': 0.8267140004437542, 'eval_runtime': 12.4236, 'eval_samples_per_second': 74.455, 'epoch': 5.997987927565393}\n",
            "{'train_runtime': 795.1332, 'train_samples_per_second': 0.936, 'epoch': 5.997987927565393}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OVwfM9ZJf12t"
      },
      "source": [
        "best_run"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vTmvFEs41WkV"
      },
      "source": [
        "#Regular Training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mNs9SDWfi7L9"
      },
      "source": [
        "training_args = TrainingArguments(\"./train\")\n",
        "training_args.evaluate_during_training = True\n",
        "training_args.adam_epsilon = 1e-8\n",
        "training_args.learning_rate = 5e-5\n",
        "training_args.fp16 = True\n",
        "training_args.per_device_train_batch_size = 16\n",
        "training_args.per_device_eval_batch_size = 16\n",
        "training_args.gradient_accumulation_steps = 2\n",
        "training_args.num_train_epochs= 8\n",
        "\n",
        "\n",
        "steps_per_epoch = (len(selected_dataset.train)// (training_args.per_device_train_batch_size * training_args.gradient_accumulation_steps))\n",
        "total_steps = steps_per_epoch * training_args.num_train_epochs\n",
        "print(steps_per_epoch)\n",
        "print(total_steps)\n",
        "#Warmup_ratio\n",
        "warmup_ratio = 0.1\n",
        "training_args.warmup_steps = total_steps*warmup_ratio\n",
        "\n",
        "training_args.evaluation_strategy = EvaluationStrategy.EPOCH\n",
        "# training_args.logging_steps = 200\n",
        "training_args.save_steps = 100000 #don't want to save any model\n",
        "training_args.seed = 42\n",
        "training_args.disable_tqdm = False\n",
        "training_args.lr_scheduler_type = 'cosine'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cACp3WCK2wr5",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2cb758d6-e1bd-4729-ea29-a2d27ad71a53"
      },
      "source": [
        "trainer = Trainer(\n",
        "    model = model_init(),\n",
        "    args = training_args,\n",
        "    train_dataset = train_dataset,\n",
        "    eval_dataset=test_dataset,\n",
        "    compute_metrics=compute_metrics,\n",
        ")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at aubmindlab/bert-base-arabertv02 were not used when initializing BertForTokenClassification: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.weight', 'cls.predictions.decoder.bias', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias']\n",
            "- This IS expected if you are initializing BertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForTokenClassification were not initialized from the model checkpoint at aubmindlab/bert-base-arabertv02 and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 428
        },
        "id": "yx336O3J2SdQ",
        "outputId": "e937906e-0954-4549-c150-4f251dbcdf10"
      },
      "source": [
        "trainer.train()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/torch/optim/lr_scheduler.py:136: UserWarning:\n",
            "\n",
            "Detected call of `lr_scheduler.step()` before `optimizer.step()`. In PyTorch 1.1.0 and later, you should call them in the opposite order: `optimizer.step()` before `lr_scheduler.step()`.  Failure to do this will result in PyTorch skipping the first value of the learning rate schedule. See more details at https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate\n",
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "        <style>\n",
              "            /* Turns off some styling */\n",
              "            progress {\n",
              "                /* gets rid of default border in Firefox and Opera. */\n",
              "                border: none;\n",
              "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
              "                background-size: auto;\n",
              "            }\n",
              "        </style>\n",
              "      \n",
              "      <progress value='992' max='992' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [992/992 06:16, Epoch 7/8]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy Score</th>\n",
              "      <th>Precision</th>\n",
              "      <th>Recall</th>\n",
              "      <th>F1</th>\n",
              "      <th>Runtime</th>\n",
              "      <th>Samples Per Second</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>No log</td>\n",
              "      <td>0.161440</td>\n",
              "      <td>0.962306</td>\n",
              "      <td>0.820141</td>\n",
              "      <td>0.766973</td>\n",
              "      <td>0.792666</td>\n",
              "      <td>8.584500</td>\n",
              "      <td>107.752000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>No log</td>\n",
              "      <td>0.145242</td>\n",
              "      <td>0.966962</td>\n",
              "      <td>0.830258</td>\n",
              "      <td>0.788436</td>\n",
              "      <td>0.808807</td>\n",
              "      <td>8.437500</td>\n",
              "      <td>109.629000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>No log</td>\n",
              "      <td>0.148088</td>\n",
              "      <td>0.969732</td>\n",
              "      <td>0.834677</td>\n",
              "      <td>0.816032</td>\n",
              "      <td>0.825249</td>\n",
              "      <td>8.470400</td>\n",
              "      <td>109.204000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>No log</td>\n",
              "      <td>0.144629</td>\n",
              "      <td>0.971499</td>\n",
              "      <td>0.840444</td>\n",
              "      <td>0.828296</td>\n",
              "      <td>0.834326</td>\n",
              "      <td>8.426100</td>\n",
              "      <td>109.779000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>0.111123</td>\n",
              "      <td>0.153060</td>\n",
              "      <td>0.970776</td>\n",
              "      <td>0.845632</td>\n",
              "      <td>0.818222</td>\n",
              "      <td>0.831701</td>\n",
              "      <td>8.732600</td>\n",
              "      <td>105.925000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.111123</td>\n",
              "      <td>0.159308</td>\n",
              "      <td>0.970896</td>\n",
              "      <td>0.844485</td>\n",
              "      <td>0.818222</td>\n",
              "      <td>0.831146</td>\n",
              "      <td>8.544100</td>\n",
              "      <td>108.262000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.111123</td>\n",
              "      <td>0.160349</td>\n",
              "      <td>0.971097</td>\n",
              "      <td>0.842318</td>\n",
              "      <td>0.821288</td>\n",
              "      <td>0.831670</td>\n",
              "      <td>8.606100</td>\n",
              "      <td>107.483000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.111123</td>\n",
              "      <td>0.160672</td>\n",
              "      <td>0.971057</td>\n",
              "      <td>0.842484</td>\n",
              "      <td>0.819974</td>\n",
              "      <td>0.831077</td>\n",
              "      <td>8.430300</td>\n",
              "      <td>109.723000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TrainOutput(global_step=992, training_loss=0.06601395337812362, metrics={'train_runtime': 376.3862, 'train_samples_per_second': 2.636, 'total_flos': 6570640586138112, 'epoch': 7.995983935742972})"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Seyz8Yaj2ZCK"
      },
      "source": [
        "trainer.save_model(\"SOME_PATH\")"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}