Arabizi_KT_CV82091_LB_8286G.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Arabizi_KT_CV82091_LB_8286G.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ojSAlH9W_gfv",
        "outputId": "6bed4e5e-d511-4ccb-e09e-179f561162cc"
      },
      "source": [
        "# Check GPU type\r\n",
        "!nvidia-smi"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Tue Mar  2 07:08:47 2021       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 460.39       Driver Version: 460.32.03    CUDA Version: 11.2     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla V100-SXM2...  Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   34C    P0    24W / 300W |      0MiB / 16160MiB |      0%      Default |\n",
            "|                               |                      |                  N/A |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5016OlJnp2kC",
        "outputId": "1310ee54-5823-4d7d-a882-3ff37756f37e"
      },
      "source": [
        "# Upgrade pip and install ktrain\r\n",
        "!pip -qq install -U pip\r\n",
        "!pip -qq install ktrain"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[K     |████████████████████████████████| 1.5MB 5.4MB/s \n",
            "\u001b[K     |████████████████████████████████| 25.3 MB 94.2 MB/s \n",
            "\u001b[K     |████████████████████████████████| 6.8 MB 61.9 MB/s \n",
            "\u001b[K     |████████████████████████████████| 981 kB 56.6 MB/s \n",
            "\u001b[K     |████████████████████████████████| 263 kB 58.2 MB/s \n",
            "\u001b[K     |████████████████████████████████| 1.3 MB 58.2 MB/s \n",
            "\u001b[K     |████████████████████████████████| 1.2 MB 60.2 MB/s \n",
            "\u001b[K     |████████████████████████████████| 468 kB 27.6 MB/s \n",
            "\u001b[K     |████████████████████████████████| 1.1 MB 60.4 MB/s \n",
            "\u001b[K     |████████████████████████████████| 883 kB 60.1 MB/s \n",
            "\u001b[K     |████████████████████████████████| 2.9 MB 65.3 MB/s \n",
            "\u001b[?25h  Building wheel for ktrain (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for seqeval (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for keras-bert (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for keras-embed-sim (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for keras-layer-normalization (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for keras-multi-head (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for keras-self-attention (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for keras-pos-embd (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for keras-position-wise-feed-forward (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for langdetect (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for syntok (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qDHDhHzWrDmm",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1c15ae83-3a35-49b0-a6e7-bd9088176a8f"
      },
      "source": [
        "!gdown --id 1LZBMbdMAr8iwmNfN2JkiFw-uGPleBOsf\r\n",
        "!unzip -q '/content/Arabizi_data.zip'"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading...\n",
            "From: https://drive.google.com/uc?id=1LZBMbdMAr8iwmNfN2JkiFw-uGPleBOsf\n",
            "To: /content/Arabizi_data.zip\n",
            "\r0.00B [00:00, ?B/s]\r3.67MB [00:00, 106MB/s]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TWw-1GHGqVI1",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "eeb20ec7-d532-422d-b491-6ae9932a9d66"
      },
      "source": [
        "# Import libaries\r\n",
        "import numpy as np \r\n",
        "import pandas as pd\r\n",
        "from tqdm import tqdm\r\n",
        "import random\r\n",
        "import os\r\n",
        "import re\r\n",
        "import ktrain\r\n",
        "from ktrain import text\r\n",
        "import tensorflow as tf\r\n",
        "from sklearn.model_selection import StratifiedKFold\r\n",
        "import string\r\n",
        "import nltk\r\n",
        "from nltk.tokenize import word_tokenize\r\n",
        "from nltk.corpus import stopwords\r\n",
        "nltk.download('punkt')\r\n",
        "import warnings\r\n",
        "warnings.filterwarnings('ignore')"
      ],
      "execution_count": 95,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gdIMbg9vnM9b"
      },
      "source": [
        "# Set seed\r\n",
        "SEED = 3031\r\n",
        "\r\n",
        "# def set_seeds(seed=SEED):\r\n",
        "#     os.environ['PYTHONHASHSEED'] = str(seed)\r\n",
        "#     random.seed(seed)\r\n",
        "#     tf.random.set_seed(seed)\r\n",
        "#     np.random.seed(seed)\r\n",
        "\r\n",
        "# def set_global_determinism(seed=SEED):\r\n",
        "#     set_seeds(seed=seed)\r\n",
        "\r\n",
        "#     os.environ['TF_DETERMINISTIC_OPS'] = '1'\r\n",
        "#     os.environ['TF_CUDNN_DETERMINISTIC'] = '1'\r\n",
        "    \r\n",
        "#     tf.config.threading.set_inter_op_parallelism_threads(1)\r\n",
        "#     tf.config.threading.set_intra_op_parallelism_threads(1)\r\n",
        "\r\n",
        "# set_global_determinism(seed=SEED)"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7rmqE-HZR3FK"
      },
      "source": [
        "def clean_text(text):\r\n",
        "    '''Make text lowercase, remove text in square brackets,remove links,remove punctuation\r\n",
        "    and remove words containing numbers.'''\r\n",
        "    text = text.lower()\r\n",
        "    text = re.sub('\\[.*?\\]', '', text)\r\n",
        "    text = re.sub('https?://\\S+|www\\.\\S+', '', text)\r\n",
        "    text = re.sub('<.*?>+', '', text)\r\n",
        "    text = re.sub('[%s]' % re.escape(string.punctuation), '', text)\r\n",
        "    text = re.sub('\\n', '', text)\r\n",
        "    text = re.sub('\\w*\\d\\w*', '', text)\r\n",
        "    return text\r\n",
        "\r\n",
        "def text_preprocessing(text):\r\n",
        "    \"\"\"\r\n",
        "    Cleaning and parsing the text.\r\n",
        "\r\n",
        "    \"\"\"\r\n",
        "    tokenizer = nltk.tokenize.RegexpTokenizer(r'\\w+')\r\n",
        "    nopunc = clean_text(text)\r\n",
        "    tokenized_text = tokenizer.tokenize(nopunc)\r\n",
        "    #remove_stopwords = [w for w in tokenized_text if w not in stopwords.words('english')]\r\n",
        "    combined_text = ' '.join(tokenized_text)\r\n",
        "    return combined_text\r\n",
        "\r\n",
        "def text_cleaner(text):\r\n",
        "  text = re.sub('\\s+',' ', text)\r\n",
        "  text = text.strip()\r\n",
        "  text = re.sub(r'(.)\\1+', r'\\1\\1', text)\r\n",
        "  return text"
      ],
      "execution_count": 77,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q0ECuC-bqVGi",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 195
        },
        "outputId": "0d391ed6-5450-4592-837e-bc5401578125"
      },
      "source": [
        "train = pd.read_csv('/content/Arabizi_data/Train.csv')\r\n",
        "test = pd.read_csv('/content/Arabizi_data/Test.csv')\r\n",
        "sample = pd.read_csv('/content/Arabizi_data/SampleSubmission.csv')\r\n",
        "train.head()"
      ],
      "execution_count": 122,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>ID</th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>13P0QT0</td>\n",
              "      <td>3sbaaaaaaaaaaaaaaaaaaaa lek ou le seim riahi o...</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>SKCLXCJ</td>\n",
              "      <td>cha3eb fey9elkoum menghir ta7ayoul ou kressi</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>V1TVXIJ</td>\n",
              "      <td>bereau degage nathef ya slim walahi ya7chiw fi...</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>U0TTYY8</td>\n",
              "      <td>ak slouma</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>68DX797</td>\n",
              "      <td>entom titmanou lina a7na 3iid moubarik a7na ch...</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "        ID                                               text  label\n",
              "0  13P0QT0  3sbaaaaaaaaaaaaaaaaaaaa lek ou le seim riahi o...     -1\n",
              "1  SKCLXCJ       cha3eb fey9elkoum menghir ta7ayoul ou kressi     -1\n",
              "2  V1TVXIJ  bereau degage nathef ya slim walahi ya7chiw fi...     -1\n",
              "3  U0TTYY8                                          ak slouma      1\n",
              "4  68DX797  entom titmanou lina a7na 3iid moubarik a7na ch...     -1"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 122
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Zmf9Co_h3Thg",
        "outputId": "caf8dd00-8415-42ed-b8a9-1c5a1d710501"
      },
      "source": [
        "train.label.value_counts()"
      ],
      "execution_count": 123,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              " 1    38239\n",
              "-1    29295\n",
              " 0     2466\n",
              "Name: label, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 123
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X6GUtTyX3cwv"
      },
      "source": [
        "train.label = train.label.astype(str)"
      ],
      "execution_count": 124,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 195
        },
        "id": "iJu-Skwq_JaM",
        "outputId": "8a4b1f2c-6b25-42a6-d177-e311e7591243"
      },
      "source": [
        "# Preview last five rows in test\r\n",
        "test.tail()"
      ],
      "execution_count": 125,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>ID</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>29995</th>\n",
              "      <td>NHXTL3R</td>\n",
              "      <td>me ihebekch raw</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>29996</th>\n",
              "      <td>U1YWB2O</td>\n",
              "      <td>nchallah rabi m3ak w iwaf9ek mais just 7abit n...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>29997</th>\n",
              "      <td>O3KYLM0</td>\n",
              "      <td>slim rabi m3ak w e5edem w 5alli l7ossed lemnay...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>29998</th>\n",
              "      <td>W4C38TY</td>\n",
              "      <td>bara 5alis rouhik yizi mitbal3it jam3iya hlaki...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>29999</th>\n",
              "      <td>4NNX5QE</td>\n",
              "      <td>rabi m3aaaak ya khawlaaa n7ebouuuuk rana barsh...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "            ID                                               text\n",
              "29995  NHXTL3R                                    me ihebekch raw\n",
              "29996  U1YWB2O  nchallah rabi m3ak w iwaf9ek mais just 7abit n...\n",
              "29997  O3KYLM0  slim rabi m3ak w e5edem w 5alli l7ossed lemnay...\n",
              "29998  W4C38TY  bara 5alis rouhik yizi mitbal3it jam3iya hlaki...\n",
              "29999  4NNX5QE  rabi m3aaaak ya khawlaaa n7ebouuuuk rana barsh..."
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 125
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gz_vbjQj3A94",
        "outputId": "455d7e45-ce09-42d8-9746-748bcfbaec6c"
      },
      "source": [
        "train.shape, test.shape, sample.shape"
      ],
      "execution_count": 126,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((70000, 3), (30000, 2), (30000, 2))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 126
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 195
        },
        "id": "bm4MnMIBSHfr",
        "outputId": "248b33be-5dff-4c5a-f7c4-5606395c68e3"
      },
      "source": [
        "tqdm.pandas()\r\n",
        "train['clean_text'] = train.text.apply(lambda x: text_cleaner(x))\r\n",
        "train.head()"
      ],
      "execution_count": 127,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>ID</th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "      <th>clean_text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>13P0QT0</td>\n",
              "      <td>3sbaaaaaaaaaaaaaaaaaaaa lek ou le seim riahi o...</td>\n",
              "      <td>-1</td>\n",
              "      <td>3sbaa lek ou le seim riahi ou 3sbaa le ca</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>SKCLXCJ</td>\n",
              "      <td>cha3eb fey9elkoum menghir ta7ayoul ou kressi</td>\n",
              "      <td>-1</td>\n",
              "      <td>cha3eb fey9elkoum menghir ta7ayoul ou kressi</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>V1TVXIJ</td>\n",
              "      <td>bereau degage nathef ya slim walahi ya7chiw fi...</td>\n",
              "      <td>-1</td>\n",
              "      <td>bereau degage nathef ya slim walahi ya7chiw fi...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>U0TTYY8</td>\n",
              "      <td>ak slouma</td>\n",
              "      <td>1</td>\n",
              "      <td>ak slouma</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>68DX797</td>\n",
              "      <td>entom titmanou lina a7na 3iid moubarik a7na ch...</td>\n",
              "      <td>-1</td>\n",
              "      <td>entom titmanou lina a7na 3iid moubarik a7na ch...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "        ID  ...                                         clean_text\n",
              "0  13P0QT0  ...          3sbaa lek ou le seim riahi ou 3sbaa le ca\n",
              "1  SKCLXCJ  ...       cha3eb fey9elkoum menghir ta7ayoul ou kressi\n",
              "2  V1TVXIJ  ...  bereau degage nathef ya slim walahi ya7chiw fi...\n",
              "3  U0TTYY8  ...                                          ak slouma\n",
              "4  68DX797  ...  entom titmanou lina a7na 3iid moubarik a7na ch...\n",
              "\n",
              "[5 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 127
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 195
        },
        "id": "ohmweWGMS6eD",
        "outputId": "d9e7865a-60ea-4a51-893e-e6f360521b31"
      },
      "source": [
        "test['clean_text'] = test.text.apply(lambda x: text_cleaner(x))\r\n",
        "test.head()"
      ],
      "execution_count": 128,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>ID</th>\n",
              "      <th>text</th>\n",
              "      <th>clean_text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2DDHQW9</td>\n",
              "      <td>barcha aaindou fiha hak w barcha teflim kadhalik</td>\n",
              "      <td>barcha aaindou fiha hak w barcha teflim kadhalik</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>5HY6UEY</td>\n",
              "      <td>ye gernabou ye 9a7ba</td>\n",
              "      <td>ye gernabou ye 9a7ba</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>ATNVUJX</td>\n",
              "      <td>saber w barra rabbi m3ak 5ouya</td>\n",
              "      <td>saber w barra rabbi m3ak 5ouya</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Q9XYVOQ</td>\n",
              "      <td>cha3ébbb ta7aaaaannnnnnnnnnn tfouuhh</td>\n",
              "      <td>cha3ébb ta7aann tfouuhh</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>TOAHLRH</td>\n",
              "      <td>rabi y5alihoulek w yfar7ek bih w inchallah itc...</td>\n",
              "      <td>rabi y5alihoulek w yfar7ek bih w inchallah itc...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "        ID  ...                                         clean_text\n",
              "0  2DDHQW9  ...   barcha aaindou fiha hak w barcha teflim kadhalik\n",
              "1  5HY6UEY  ...                               ye gernabou ye 9a7ba\n",
              "2  ATNVUJX  ...                     saber w barra rabbi m3ak 5ouya\n",
              "3  Q9XYVOQ  ...                            cha3ébb ta7aann tfouuhh\n",
              "4  TOAHLRH  ...  rabi y5alihoulek w yfar7ek bih w inchallah itc...\n",
              "\n",
              "[5 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 128
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h5dscE0Vre-J",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9f896752-11e1-4e1a-efac-ab3d03d9d7d6"
      },
      "source": [
        "MODEL_NAME = 'bert-base-uncased'\r\n",
        "MAX_LEN = 64\r\n",
        "BATCH_SIZE = 64\r\n",
        "FOLDS = 5\r\n",
        "LR = 3e-5\r\n",
        "EPOCHS = 3\r\n",
        "\r\n",
        "# List of class names\r\n",
        "CLASS_NAMES = sorted(train.label.unique().tolist()) # ['afya', 'burudani', 'kimataifa', 'kitaifa', 'michezo', 'uchumi']\r\n",
        "\r\n",
        "# Instantiate transformer with the provided parameters\r\n",
        "t = text.Transformer(model_name=MODEL_NAME, maxlen=MAX_LEN, class_names=CLASS_NAMES, batch_size=BATCH_SIZE)\r\n",
        "CLASS_NAMES"
      ],
      "execution_count": 129,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['-1', '0', '1']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 129
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "chBN-wZiy1QL",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ff63f6ef-2b95-47b3-c33d-391eb8bac3b8"
      },
      "source": [
        "%%time\r\n",
        "# Prepare test data\r\n",
        "test_data = np.asarray(test.clean_text)\r\n",
        "\r\n",
        "# Set number of folds to 3\r\n",
        "folds = StratifiedKFold(n_splits=FOLDS, random_state=SEED, shuffle=False)\r\n",
        "\r\n",
        "# List to store predictions and loss-score per fold\r\n",
        "oof_preds = []\r\n",
        "oof_loss_score = []\r\n",
        "\r\n",
        "for i, (train_index, test_index) in enumerate(folds.split(train.clean_text, train.label)):\r\n",
        "  X_train, X_test = list(train.loc[train_index, 'clean_text']), list(train.loc[test_index, 'clean_text'])\r\n",
        "  y_train, y_test = np.asarray(train.loc[train_index, 'label']), np.asarray(train.loc[test_index, 'label'])\r\n",
        "\r\n",
        "  # Preprocess training and validation data\r\n",
        "  train_set = t.preprocess_train(X_train, y_train, verbose = 0)\r\n",
        "  val_set = t.preprocess_test(X_test, y_test, verbose = 0)\r\n",
        "\r\n",
        "  # Instantiate model\r\n",
        "  model = t.get_classifier()\r\n",
        "  learner = ktrain.get_learner(model, train_data=train_set, val_data=val_set, batch_size=BATCH_SIZE)\r\n",
        "\r\n",
        "  history = learner.fit(LR, n_cycles=EPOCHS, checkpoint_folder='/tmp')\r\n",
        "  fold_accuracies = history.history['val_accuracy'] \r\n",
        "  best_score, best_epoch = max(fold_accuracies), np.array(fold_accuracies).argmax() + 1\r\n",
        "  oof_loss_score.append(best_score)\r\n",
        "  print(f'\\033[1m\\033[92m Fold {i+1}: {best_score}\\33[0m\\n')\r\n",
        "\r\n",
        "  #Load best weights\r\n",
        "  model = t.get_classifier()\r\n",
        "  model.load_weights('../tmp/weights-0' + str(best_epoch) + '.hdf5')\r\n",
        "  learner = ktrain.get_learner(model, train_data=train_set, val_data=val_set, batch_size=BATCH_SIZE)\r\n",
        "\r\n",
        "  # Make predictions\r\n",
        "  preds = ktrain.get_predictor(learner.model, preproc=t).predict(test_data, return_proba=True)\r\n",
        "\r\n",
        "  # Append preds to oof_preds list\r\n",
        "  oof_preds.append(preds)\r\n",
        "\r\n",
        "# Check cv score and prepare submission file\r\n",
        "LOSS = np.round(np.mean(oof_loss_score), 5)\r\n",
        "print(f'\\n\\33[96m\\33[1m\\33[4m Mean Loss: {LOSS}\\33[0m')\r\n",
        "\r\n",
        "name = f'{MODEL_NAME}_ML{MAX_LEN}_BS{BATCH_SIZE}_FD{FOLDS}_LR{LR}_EP{EPOCHS}_LS{LOSS}'\r\n",
        "sub = pd.DataFrame(np.mean(oof_preds, axis=0), columns = t.get_classes())\r\n",
        "sub.to_csv(name + '.csv', index = False)\r\n",
        "ss = pd.DataFrame({'ID':test.ID, 'label': sub.idxmax(axis = 1)})\r\n",
        "ss.to_csv(f'KT_bert{LOSS}.csv', index = False)"
      ],
      "execution_count": 130,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/3\n",
            "875/875 [==============================] - 292s 305ms/step - loss: 0.6345 - accuracy: 0.7121 - val_loss: 0.4663 - val_accuracy: 0.8023\n",
            "Epoch 2/3\n",
            "875/875 [==============================] - 273s 301ms/step - loss: 0.4152 - accuracy: 0.8300 - val_loss: 0.4754 - val_accuracy: 0.8016\n",
            "Epoch 3/3\n",
            "875/875 [==============================] - 274s 301ms/step - loss: 0.3116 - accuracy: 0.8760 - val_loss: 0.4404 - val_accuracy: 0.8218\n",
            "\u001b[1m\u001b[92m Fold 1: 0.8217856884002686\u001b[0m\n",
            "\n",
            "Epoch 1/3\n",
            "875/875 [==============================] - 291s 305ms/step - loss: 0.6398 - accuracy: 0.7110 - val_loss: 0.4818 - val_accuracy: 0.8003\n",
            "Epoch 2/3\n",
            "875/875 [==============================] - 274s 302ms/step - loss: 0.4155 - accuracy: 0.8297 - val_loss: 0.4434 - val_accuracy: 0.8172\n",
            "Epoch 3/3\n",
            "875/875 [==============================] - 274s 302ms/step - loss: 0.3036 - accuracy: 0.8799 - val_loss: 0.4486 - val_accuracy: 0.8184\n",
            "\u001b[1m\u001b[92m Fold 2: 0.8184285759925842\u001b[0m\n",
            "\n",
            "Epoch 1/3\n",
            "875/875 [==============================] - 292s 305ms/step - loss: 0.6423 - accuracy: 0.7103 - val_loss: 0.4753 - val_accuracy: 0.8006\n",
            "Epoch 2/3\n",
            "875/875 [==============================] - 274s 302ms/step - loss: 0.4082 - accuracy: 0.8323 - val_loss: 0.4544 - val_accuracy: 0.8154\n",
            "Epoch 3/3\n",
            "875/875 [==============================] - 275s 303ms/step - loss: 0.3061 - accuracy: 0.8820 - val_loss: 0.5234 - val_accuracy: 0.8090\n",
            "\u001b[1m\u001b[92m Fold 3: 0.8153571486473083\u001b[0m\n",
            "\n",
            "Epoch 1/3\n",
            "875/875 [==============================] - 291s 305ms/step - loss: 0.6313 - accuracy: 0.7151 - val_loss: 0.4684 - val_accuracy: 0.7984\n",
            "Epoch 2/3\n",
            "875/875 [==============================] - 274s 302ms/step - loss: 0.4189 - accuracy: 0.8271 - val_loss: 0.4292 - val_accuracy: 0.8226\n",
            "Epoch 3/3\n",
            "875/875 [==============================] - 274s 302ms/step - loss: 0.3045 - accuracy: 0.8777 - val_loss: 0.4774 - val_accuracy: 0.8154\n",
            "\u001b[1m\u001b[92m Fold 4: 0.8226428627967834\u001b[0m\n",
            "\n",
            "Epoch 1/3\n",
            "875/875 [==============================] - 294s 306ms/step - loss: 0.6402 - accuracy: 0.7073 - val_loss: 0.4774 - val_accuracy: 0.8046\n",
            "Epoch 2/3\n",
            "875/875 [==============================] - 275s 303ms/step - loss: 0.4161 - accuracy: 0.8290 - val_loss: 0.4452 - val_accuracy: 0.8175\n",
            "Epoch 3/3\n",
            "875/875 [==============================] - 275s 303ms/step - loss: 0.3071 - accuracy: 0.8792 - val_loss: 0.4636 - val_accuracy: 0.8264\n",
            "\u001b[1m\u001b[92m Fold 5: 0.8263571262359619\u001b[0m\n",
            "\n",
            "\n",
            "\u001b[96m\u001b[1m\u001b[4m Mean Loss: 0.82091\u001b[0m\n",
            "CPU times: user 35min 7s, sys: 9min 30s, total: 44min 38s\n",
            "Wall time: 1h 17min 35s\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}