disaster_nlp.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/MathBorgess/into_pytorch/blob/master/disaster_nlp.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 91
        },
        "id": "8UJDFz8KwlLx",
        "outputId": "6a95c2d8-b6a1-4dd6-ad90-e485eb694b8e"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-8dc3a91e-1de9-40fe-b6de-41393eb551eb\" name=\"files[]\" multiple disabled\n",
              "        style=\"border:none\" />\n",
              "     <output id=\"result-8dc3a91e-1de9-40fe-b6de-41393eb551eb\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script>// Copyright 2017 Google LLC\n",
              "//\n",
              "// Licensed under the Apache License, Version 2.0 (the \"License\");\n",
              "// you may not use this file except in compliance with the License.\n",
              "// You may obtain a copy of the License at\n",
              "//\n",
              "//      http://www.apache.org/licenses/LICENSE-2.0\n",
              "//\n",
              "// Unless required by applicable law or agreed to in writing, software\n",
              "// distributed under the License is distributed on an \"AS IS\" BASIS,\n",
              "// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
              "// See the License for the specific language governing permissions and\n",
              "// limitations under the License.\n",
              "\n",
              "/**\n",
              " * @fileoverview Helpers for google.colab Python module.\n",
              " */\n",
              "(function(scope) {\n",
              "function span(text, styleAttributes = {}) {\n",
              "  const element = document.createElement('span');\n",
              "  element.textContent = text;\n",
              "  for (const key of Object.keys(styleAttributes)) {\n",
              "    element.style[key] = styleAttributes[key];\n",
              "  }\n",
              "  return element;\n",
              "}\n",
              "\n",
              "// Max number of bytes which will be uploaded at a time.\n",
              "const MAX_PAYLOAD_SIZE = 100 * 1024;\n",
              "\n",
              "function _uploadFiles(inputId, outputId) {\n",
              "  const steps = uploadFilesStep(inputId, outputId);\n",
              "  const outputElement = document.getElementById(outputId);\n",
              "  // Cache steps on the outputElement to make it available for the next call\n",
              "  // to uploadFilesContinue from Python.\n",
              "  outputElement.steps = steps;\n",
              "\n",
              "  return _uploadFilesContinue(outputId);\n",
              "}\n",
              "\n",
              "// This is roughly an async generator (not supported in the browser yet),\n",
              "// where there are multiple asynchronous steps and the Python side is going\n",
              "// to poll for completion of each step.\n",
              "// This uses a Promise to block the python side on completion of each step,\n",
              "// then passes the result of the previous step as the input to the next step.\n",
              "function _uploadFilesContinue(outputId) {\n",
              "  const outputElement = document.getElementById(outputId);\n",
              "  const steps = outputElement.steps;\n",
              "\n",
              "  const next = steps.next(outputElement.lastPromiseValue);\n",
              "  return Promise.resolve(next.value.promise).then((value) => {\n",
              "    // Cache the last promise value to make it available to the next\n",
              "    // step of the generator.\n",
              "    outputElement.lastPromiseValue = value;\n",
              "    return next.value.response;\n",
              "  });\n",
              "}\n",
              "\n",
              "/**\n",
              " * Generator function which is called between each async step of the upload\n",
              " * process.\n",
              " * @param {string} inputId Element ID of the input file picker element.\n",
              " * @param {string} outputId Element ID of the output display.\n",
              " * @return {!Iterable<!Object>} Iterable of next steps.\n",
              " */\n",
              "function* uploadFilesStep(inputId, outputId) {\n",
              "  const inputElement = document.getElementById(inputId);\n",
              "  inputElement.disabled = false;\n",
              "\n",
              "  const outputElement = document.getElementById(outputId);\n",
              "  outputElement.innerHTML = '';\n",
              "\n",
              "  const pickedPromise = new Promise((resolve) => {\n",
              "    inputElement.addEventListener('change', (e) => {\n",
              "      resolve(e.target.files);\n",
              "    });\n",
              "  });\n",
              "\n",
              "  const cancel = document.createElement('button');\n",
              "  inputElement.parentElement.appendChild(cancel);\n",
              "  cancel.textContent = 'Cancel upload';\n",
              "  const cancelPromise = new Promise((resolve) => {\n",
              "    cancel.onclick = () => {\n",
              "      resolve(null);\n",
              "    };\n",
              "  });\n",
              "\n",
              "  // Wait for the user to pick the files.\n",
              "  const files = yield {\n",
              "    promise: Promise.race([pickedPromise, cancelPromise]),\n",
              "    response: {\n",
              "      action: 'starting',\n",
              "    }\n",
              "  };\n",
              "\n",
              "  cancel.remove();\n",
              "\n",
              "  // Disable the input element since further picks are not allowed.\n",
              "  inputElement.disabled = true;\n",
              "\n",
              "  if (!files) {\n",
              "    return {\n",
              "      response: {\n",
              "        action: 'complete',\n",
              "      }\n",
              "    };\n",
              "  }\n",
              "\n",
              "  for (const file of files) {\n",
              "    const li = document.createElement('li');\n",
              "    li.append(span(file.name, {fontWeight: 'bold'}));\n",
              "    li.append(span(\n",
              "        `(${file.type || 'n/a'}) - ${file.size} bytes, ` +\n",
              "        `last modified: ${\n",
              "            file.lastModifiedDate ? file.lastModifiedDate.toLocaleDateString() :\n",
              "                                    'n/a'} - `));\n",
              "    const percent = span('0% done');\n",
              "    li.appendChild(percent);\n",
              "\n",
              "    outputElement.appendChild(li);\n",
              "\n",
              "    const fileDataPromise = new Promise((resolve) => {\n",
              "      const reader = new FileReader();\n",
              "      reader.onload = (e) => {\n",
              "        resolve(e.target.result);\n",
              "      };\n",
              "      reader.readAsArrayBuffer(file);\n",
              "    });\n",
              "    // Wait for the data to be ready.\n",
              "    let fileData = yield {\n",
              "      promise: fileDataPromise,\n",
              "      response: {\n",
              "        action: 'continue',\n",
              "      }\n",
              "    };\n",
              "\n",
              "    // Use a chunked sending to avoid message size limits. See b/62115660.\n",
              "    let position = 0;\n",
              "    do {\n",
              "      const length = Math.min(fileData.byteLength - position, MAX_PAYLOAD_SIZE);\n",
              "      const chunk = new Uint8Array(fileData, position, length);\n",
              "      position += length;\n",
              "\n",
              "      const base64 = btoa(String.fromCharCode.apply(null, chunk));\n",
              "      yield {\n",
              "        response: {\n",
              "          action: 'append',\n",
              "          file: file.name,\n",
              "          data: base64,\n",
              "        },\n",
              "      };\n",
              "\n",
              "      let percentDone = fileData.byteLength === 0 ?\n",
              "          100 :\n",
              "          Math.round((position / fileData.byteLength) * 100);\n",
              "      percent.textContent = `${percentDone}% done`;\n",
              "\n",
              "    } while (position < fileData.byteLength);\n",
              "  }\n",
              "\n",
              "  // All done.\n",
              "  yield {\n",
              "    response: {\n",
              "      action: 'complete',\n",
              "    }\n",
              "  };\n",
              "}\n",
              "\n",
              "scope.google = scope.google || {};\n",
              "scope.google.colab = scope.google.colab || {};\n",
              "scope.google.colab._files = {\n",
              "  _uploadFiles,\n",
              "  _uploadFilesContinue,\n",
              "};\n",
              "})(self);\n",
              "</script> "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving kaggle.json to kaggle (2).json\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'kaggle (2).json': b'{\"username\":\"matheusborgess\",\"key\":\"c60158f16484abb1f41fbffa330bcc22\"}'}"
            ]
          },
          "metadata": {},
          "execution_count": 1
        }
      ],
      "source": [
        "from google.colab import files\n",
        "files.upload()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5mAXd__dv8Us",
        "outputId": "4d66c053-3d46-4664-dd20-d9d4373ecf33"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: transformers in /usr/local/lib/python3.10/dist-packages (4.31.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.12.2)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.14.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.16.4)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (1.23.5)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (23.1)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2023.6.3)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.31.0)\n",
            "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.13.3)\n",
            "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.3.2)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.66.1)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.14.1->transformers) (2023.6.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.14.1->transformers) (4.7.1)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.2.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.4)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2023.7.22)\n",
            "nlp-getting-started.zip: Skipping, found more recently modified local copy (use --force to force download)\n"
          ]
        }
      ],
      "source": [
        "%pip install -q kaggle\n",
        "%pip install transformers\n",
        "!mkdir -p ~/.kaggle\n",
        "!cp kaggle.json ~/.kaggle/\n",
        "!chmod 600 ~/.kaggle/kaggle.json\n",
        "\n",
        "!kaggle competitions download -c nlp-getting-started"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "LUrcWsGew67F"
      },
      "outputs": [],
      "source": [
        "import zipfile\n",
        "with zipfile.ZipFile(\"nlp-getting-started.zip\",\"r\") as zip_ref:\n",
        "    zip_ref.extractall(\"data\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "0uY5248JxSRW",
        "outputId": "3b786762-553e-4143-d965-2341d026485c"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   id keyword location                                               text  \\\n",
              "0   1     NaN      NaN  Our Deeds are the Reason of this #earthquake M...   \n",
              "1   4     NaN      NaN             Forest fire near La Ronge Sask. Canada   \n",
              "2   5     NaN      NaN  All residents asked to 'shelter in place' are ...   \n",
              "3   6     NaN      NaN  13,000 people receive #wildfires evacuation or...   \n",
              "4   7     NaN      NaN  Just got sent this photo from Ruby #Alaska as ...   \n",
              "\n",
              "   target  \n",
              "0       1  \n",
              "1       1  \n",
              "2       1  \n",
              "3       1  \n",
              "4       1  "
            ],
            "text/html": [
              "\n",
              "\n",
              "  <div id=\"df-19bcef4a-c2e1-4a14-9e50-005a4bd2fe67\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>keyword</th>\n",
              "      <th>location</th>\n",
              "      <th>text</th>\n",
              "      <th>target</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Our Deeds are the Reason of this #earthquake M...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>4</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Forest fire near La Ronge Sask. Canada</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>5</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>All residents asked to 'shelter in place' are ...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>6</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>13,000 people receive #wildfires evacuation or...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>7</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Just got sent this photo from Ruby #Alaska as ...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-19bcef4a-c2e1-4a14-9e50-005a4bd2fe67')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "\n",
              "\n",
              "\n",
              "    <div id=\"df-f9b0ec8d-79fb-4920-82a7-6c15a4fb808a\">\n",
              "      <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-f9b0ec8d-79fb-4920-82a7-6c15a4fb808a')\"\n",
              "              title=\"Suggest charts.\"\n",
              "              style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "      </button>\n",
              "    </div>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "    background-color: #E8F0FE;\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: #1967D2;\n",
              "    height: 32px;\n",
              "    padding: 0 0 0 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: #E2EBFA;\n",
              "    box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: #174EA6;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "    background-color: #3B4455;\n",
              "    fill: #D2E3FC;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart:hover {\n",
              "    background-color: #434B5C;\n",
              "    box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "    filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "    fill: #FFFFFF;\n",
              "  }\n",
              "</style>\n",
              "\n",
              "    <script>\n",
              "      async function quickchart(key) {\n",
              "        const containerElement = document.querySelector('#' + key);\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      }\n",
              "    </script>\n",
              "\n",
              "      <script>\n",
              "\n",
              "function displayQuickchartButton(domScope) {\n",
              "  let quickchartButtonEl =\n",
              "    domScope.querySelector('#df-f9b0ec8d-79fb-4920-82a7-6c15a4fb808a button.colab-df-quickchart');\n",
              "  quickchartButtonEl.style.display =\n",
              "    google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "}\n",
              "\n",
              "        displayQuickchartButton(document);\n",
              "      </script>\n",
              "      <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-19bcef4a-c2e1-4a14-9e50-005a4bd2fe67 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-19bcef4a-c2e1-4a14-9e50-005a4bd2fe67');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ],
      "source": [
        "import pandas as pd\n",
        "train_set = pd.read_csv('data/train.csv')\n",
        "train_set.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "HZtveKdtznjP"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "from torch import nn\n",
        "from torch import optim\n",
        "from transformers import BertTokenizer, BertModel\n",
        "from torch.utils.data import DataLoader, Dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "gAn99tNuDcOc"
      },
      "outputs": [],
      "source": [
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n",
        "\n",
        "max_input_length = tokenizer.max_model_input_sizes['bert-base-uncased']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "QWjBsad2DcOd"
      },
      "outputs": [],
      "source": [
        "class DisasterAnalysisDataset(Dataset):\n",
        "    def __init__(self, dataframe, tokenizer, max_length):\n",
        "        self.data = dataframe\n",
        "        self.tokenizer = tokenizer\n",
        "        self.max_length = max_length\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.data)\n",
        "\n",
        "    def __getitem__(self, index):\n",
        "        text = str(self.data.loc[index, 'text']) + str(self.data.loc[index, 'location']) if self.data.loc[index, 'location'] else '' + str(self.data.loc[index, 'keyword']) if self.data.loc[index, 'keyword'] else ''\n",
        "        label = self.data.loc[index, 'target']\n",
        "\n",
        "        encoding = self.tokenizer.encode_plus(\n",
        "            text,\n",
        "            add_special_tokens=True,\n",
        "            max_length=self.max_length,\n",
        "            padding='max_length',\n",
        "            truncation=True,\n",
        "            return_tensors='pt',\n",
        "            return_attention_mask=True\n",
        "        )\n",
        "\n",
        "        input_ids = encoding['input_ids'].flatten()\n",
        "        attention_mask = encoding['attention_mask'].flatten()\n",
        "\n",
        "        return {\n",
        "            'input_ids': input_ids,\n",
        "            'attention_mask': attention_mask,\n",
        "            'labels': torch.tensor(label, dtype=torch.float)\n",
        "        }\n",
        "\n",
        "batch_size = 128\n",
        "\n",
        "dataset = DisasterAnalysisDataset(train_set, tokenizer, max_input_length)\n",
        "dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "DtL97Wq_DcOe"
      },
      "outputs": [],
      "source": [
        "bert = BertModel.from_pretrained('bert-base-uncased')\n",
        "\n",
        "class DisasterAnalysisModel(nn.Module):\n",
        "    def __init__(self, bert, hidden_dim, output_dim, n_layers, bidirectional, dropout):\n",
        "        super(DisasterAnalysisModel,self).__init__()\n",
        "        self.bert = bert\n",
        "        embedding_dim = bert.config.to_dict()['hidden_size']\n",
        "\n",
        "        self.rnn = nn.GRU(embedding_dim,\n",
        "                          hidden_dim,\n",
        "                          num_layers = n_layers,\n",
        "                          bidirectional = bidirectional,\n",
        "                          batch_first = True,\n",
        "                          dropout = 0 if n_layers < 2 else dropout)\n",
        "\n",
        "        self.out = nn.Linear(hidden_dim * 2 if bidirectional else hidden_dim, output_dim)\n",
        "\n",
        "        self.dropout = nn.Dropout(dropout)\n",
        "\n",
        "    def forward(self, text):\n",
        "\n",
        "        #text = [batch size, sent len]\n",
        "\n",
        "        with torch.no_grad():\n",
        "            embedded = self.bert(text)[0]\n",
        "\n",
        "        #embedded = [batch size, sent len, emb dim]\n",
        "\n",
        "        _, hidden = self.rnn(embedded)\n",
        "\n",
        "        #hidden = [n layers * n directions, batch size, emb dim]\n",
        "\n",
        "        if self.rnn.bidirectional:\n",
        "            hidden = self.dropout(torch.cat((hidden[-2,:,:], hidden[-1,:,:]), dim = 1))\n",
        "        else:\n",
        "            hidden = self.dropout(hidden[-1,:,:])\n",
        "\n",
        "        #hidden = [batch size, hid dim]\n",
        "\n",
        "        output = self.out(hidden)\n",
        "\n",
        "        #output = [batch size, out dim]\n",
        "\n",
        "        return output"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "JOmZ_RwYDcOf"
      },
      "outputs": [],
      "source": [
        "DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
        "\n",
        "HIDDEN_DIM = 256\n",
        "OUTPUT_DIM = 1\n",
        "N_LAYERS = 2\n",
        "BIDIRECTIONAL = True\n",
        "DROPOUT = 0.25\n",
        "model = DisasterAnalysisModel(bert,\n",
        "                            HIDDEN_DIM,\n",
        "                            OUTPUT_DIM,\n",
        "                            N_LAYERS,\n",
        "                            BIDIRECTIONAL,\n",
        "                            DROPOUT).to(DEVICE)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "5iW2oC5pDcOf"
      },
      "outputs": [],
      "source": [
        "optimizer = optim.Adam(model.parameters())\n",
        "criterion = nn.BCEWithLogitsLoss().to(DEVICE)\n",
        "def binary_accuracy(preds, y):\n",
        "    correct = (torch.round(torch.sigmoid(preds)) == y).float()\n",
        "    acc = correct.sum() / len(correct)\n",
        "    return acc"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "pdzNvMo3DcOg"
      },
      "outputs": [],
      "source": [
        "def train(model, iterator, optimizer: optim.Adam, criterion: nn.BCEWithLogitsLoss, device):\n",
        "    epoch_loss = 0\n",
        "    epoch_acc = 0\n",
        "\n",
        "    model.train()\n",
        "\n",
        "    for batch in iterator:\n",
        "        optimizer.zero_grad()\n",
        "        predictions = model(batch['input_ids'].to(device)).squeeze(1)\n",
        "\n",
        "        loss = criterion(predictions, batch['labels'].to(device))\n",
        "\n",
        "        acc = binary_accuracy(predictions, batch['labels'].to(device))\n",
        "\n",
        "        loss.backward()\n",
        "\n",
        "        optimizer.step()\n",
        "\n",
        "        epoch_loss += loss.item()\n",
        "        epoch_acc += acc.item()\n",
        "\n",
        "    return epoch_loss / len(iterator), epoch_acc / len(iterator)\n",
        "\n",
        "def evaluate(model, iterator, criterion, device):\n",
        "    model.eval()\n",
        "    predictions = []\n",
        "    with torch.no_grad():\n",
        "\n",
        "        for batch in iterator:\n",
        "            predictions.append(model(batch['input_ids'].to(device)).squeeze(1))\n",
        "\n",
        "    return predictions"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "chRJQ3apDcOh",
        "outputId": "decf23c0-e0b4-4936-f8e4-2cc53c343d54"
      },
      "outputs": [
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "We strongly recommend passing in an `attention_mask` since your input_ids may be padded. See https://huggingface.co/docs/transformers/troubleshooting#incorrect-output-when-padding-tokens-arent-masked.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 01 | Epoch Time: 287.3209114074707s\n",
            "\tTrain Loss: 0.538 | Train Acc: 73.67%\n",
            "Epoch: 02 | Epoch Time: 295.88932037353516s\n",
            "\tTrain Loss: 0.441 | Train Acc: 79.94%\n",
            "Epoch: 03 | Epoch Time: 296.71810817718506s\n",
            "\tTrain Loss: 0.418 | Train Acc: 81.47%\n",
            "Epoch: 04 | Epoch Time: 297.09863662719727s\n",
            "\tTrain Loss: 0.399 | Train Acc: 82.77%\n",
            "Epoch: 05 | Epoch Time: 296.5816104412079s\n",
            "\tTrain Loss: 0.386 | Train Acc: 83.23%\n"
          ]
        }
      ],
      "source": [
        "import time\n",
        "EPOCHS = 5\n",
        "\n",
        "best_valid_loss = float('inf')\n",
        "\n",
        "for epoch in range(EPOCHS):\n",
        "\n",
        "    start_time = time.time()\n",
        "\n",
        "    train_loss, train_acc = train(model, dataloader, optimizer, criterion, DEVICE)\n",
        "\n",
        "    end_time = time.time()\n",
        "\n",
        "    print(f'Epoch: {epoch+1:02} | Epoch Time: {end_time - start_time}s')\n",
        "    print(f'\\tTrain Loss: {train_loss:.3f} | Train Acc: {train_acc*100:.2f}%')"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def predict_sentiment(model, tokenizer, max_length, device):\n",
        "    model.eval()\n",
        "    return lambda sentence_list : [torch.sigmoid(model(torch.LongTensor(tokenizer.encode_plus(\n",
        "                  sentence,\n",
        "                  add_special_tokens=True,\n",
        "                  max_length= max_length,\n",
        "                  padding='max_length',\n",
        "                  truncation=True,\n",
        "                  return_tensors='pt',\n",
        "                  return_attention_mask=True\n",
        "              )['input_ids'].flatten()).to(device).unsqueeze(0))).item() for sentence in sentence_list]"
      ],
      "metadata": {
        "id": "6Kmt-sylRiSN"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "predict_sentiment(model, tokenizer, max_input_length, DEVICE)([\"I was at home, and the earthquake started, but it already stopped\", \"everything is fine\"])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iNFxaCImSTbL",
        "outputId": "58895925-299a-48c6-dca2-6240cf7d90ca"
      },
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.9810390472412109, 0.043908510357141495]"
            ]
          },
          "metadata": {},
          "execution_count": 24
        }
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.11.3"
    },
    "accelerator": "GPU"
  },
  "nbformat": 4,
  "nbformat_minor": 0
}