Paper's_code_Seq2seq(pytorch).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "EhayRD-u5M43",
    "outputId": "1746f2a8-2727-41bc-9797-1ddabce9c979"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mounted at /content/drive\n"
     ]
    }
   ],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/drive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "rqTONwd35QnJ",
    "outputId": "f91fd1b1-b6ef-4deb-bcb5-714126c2eb24"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/content/drive/MyDrive/paper codes\n"
     ]
    }
   ],
   "source": [
    "%cd /content/drive/MyDrive/paper codes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "id": "kLAyafWf5djv",
    "outputId": "66fe7e81-3f67-4c86-958f-691a9cd31fb9"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.google.colaboratory.intrinsic+json": {
       "type": "string"
      },
      "text/plain": [
       "'/content/drive/MyDrive/paper codes'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%pwd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Xku24IQK5non"
   },
   "source": [
    "## Data 준비\n",
    "- torch 버전과 torchtext버전이 잘 맞아야함,,\n",
    "- 여기서는 torch 1.9와 torchtext 0.10을 사용하였음"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "zVPdn-2U9MFO"
   },
   "outputs": [],
   "source": [
    "!pip install torch==1.9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "I_My6qgV59Ge"
   },
   "outputs": [],
   "source": [
    "!pip install torchtext==0.10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "3K7pbLas5mN-"
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "\n",
    "from torchtext.legacy.datasets import Multi30k\n",
    "from torchtext.legacy.data import Field, BucketIterator\n",
    "\n",
    "import spacy\n",
    "import numpy as np\n",
    "\n",
    "import random\n",
    "import math\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "id": "Vq3bx-vhCvKa"
   },
   "outputs": [],
   "source": [
    "import torchtext"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "juiWswkcCjyn",
    "outputId": "337f53cb-49bb-489b-87ba-c341cc95b3c6"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torchtext version: 0.10.0\n",
      "pytorch version: 1.9.0+cu102\n"
     ]
    }
   ],
   "source": [
    "print(f\"torchtext version: {torchtext.__version__}\")\n",
    "print(f\"pytorch version: {torch.__version__}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "mTvONdrJ-9am"
   },
   "outputs": [],
   "source": [
    "!python -m spacy download en_core_web_sm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "pw8LXOOG__Wr",
    "outputId": "7b8a45a3-45cb-495f-f004-634b4965d6b5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Installing collected packages: de-core-news-sm\n",
      "Successfully installed de-core-news-sm-3.4.0\n",
      "\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n",
      "You can now load the package via spacy.load('de_core_news_sm')\n"
     ]
    }
   ],
   "source": [
    "!python -m spacy download de_core_news_sm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "HCgmoJYD5qPx"
   },
   "outputs": [],
   "source": [
    "#Tokenizer 정의\n",
    "spacy_de = spacy.load('de_core_news_sm')\n",
    "spacy_en = spacy.load('en_core_web_sm')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "id": "lCrm3_UABbEL"
   },
   "outputs": [],
   "source": [
    "#Source sentece를 뒤집어서 학습시키는 것이 효과적임을 보였음.\n",
    "def tokenize_de(text):\n",
    "    return [tok.text for tok in spacy_de.tokenizer(text)][::-1]\n",
    "\n",
    "def tokenize_en(text):\n",
    "    return [tok.text for tok in spacy_en.tokenizer(text)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "id": "eVSIeOVwCM1g"
   },
   "outputs": [],
   "source": [
    "#Field는 아마 버전에 따라서 import하는 방식이 다를 것이다.\n",
    "        #Torchtext : 0.10.0, Torch:1.9.0+cu102\n",
    "#어쨌든 Fiel는 데이터가 어떻게 전처리되는지 파이프라인을 한번에 확인할 수 있다.\n",
    "SRC = Field(tokenize = tokenize_de,\n",
    "            init_token = '',\n",
    "            eos_token = '',\n",
    "            lower = True)\n",
    "\n",
    "TRG = Field(tokenize = tokenize_en,\n",
    "            init_token = '',\n",
    "            eos_token = '',\n",
    "            lower = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "id": "HlcqhsKhDL-_"
   },
   "outputs": [],
   "source": [
    "#데이터 다운받기 << Multi30k dataset\n",
    "    #3만개의 영어, 독일어 문장 쌍\n",
    "    #각 문장은 최대 12개의 단어를 가지고 있음\n",
    "train_data, valid_data, test_data = Multi30k.splits(exts = ('.de', '.en'), \n",
    "                                         #exts는 어떤 언어를 source와 target으로 할 지 결정한다\n",
    "                                         fields = (SRC, TRG))\n",
    "                                        #field를 통해 한번에 전처리 진행"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Bx1HwUH4Dxno",
    "outputId": "42ddfd51-eca3-432c-8bf0-b720f6ae2248"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of training examples: 29000\n",
      "Number of validation examples: 1014\n",
      "Number of testing examples: 1000\n"
     ]
    }
   ],
   "source": [
    "#데이터의 크기를 확인\n",
    "print(f\"Number of training examples: {len(train_data.examples)}\")\n",
    "print(f\"Number of validation examples: {len(valid_data.examples)}\")\n",
    "print(f\"Number of testing examples: {len(test_data.examples)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "myKwvOtyESgd",
    "outputId": "a4f4d8ca-978f-4ac1-bd76-05c628c86162"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'src': ['.', 'büsche', 'vieler', 'nähe', 'der', 'in', 'freien', 'im', 'sind', 'männer', 'weiße', 'junge', 'zwei'], 'trg': ['two', 'young', ',', 'white', 'males', 'are', 'outside', 'near', 'many', 'bushes', '.']}\n"
     ]
    }
   ],
   "source": [
    "print(vars(train_data.examples[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "id": "_xkSCXPKEUOs"
   },
   "outputs": [],
   "source": [
    "SRC.build_vocab(train_data, min_freq = 2)\n",
    "TRG.build_vocab(train_data, min_freq = 2) #min_freq: 최소 등장 조건\n",
    "            #build_vocab --> vectors = '' 으로 사전 훈련된 임베딩 벡터도 설정할 수 있음!\n",
    "                            #unit_init << 사전훈련된 단어 집합에 없는 단어를 0으로 만드는 것을 방지하기 위한 파라미터"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "TLg3HHPYEdRr",
    "outputId": "de699aab-bf3d-47ae-9512-e5199936bfe3"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique tokens in source (de) vocabulary: 7852\n",
      "Unique tokens in target (en) vocabulary: 5892\n"
     ]
    }
   ],
   "source": [
    "print(f\"Unique tokens in source (de) vocabulary: {len(SRC.vocab)}\")\n",
    "print(f\"Unique tokens in target (en) vocabulary: {len(TRG.vocab)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "id": "hXZYX3p2EgI3"
   },
   "outputs": [],
   "source": [
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "id": "8C_LduTYFG7w"
   },
   "outputs": [],
   "source": [
    "BATCH_SIZE  = 128\n",
    "\n",
    "train_iterator, valid_iterator, test_iterator = BucketIterator.splits(\n",
    "    (train_data, valid_data, test_data),#data\n",
    "    batch_size = BATCH_SIZE, \n",
    "    device = device)\n",
    "#BucketIterator << NLP dataset에 특화된 iterator\n",
    "    #전처리 파이프라인을 통과하게 할 수 있고, \n",
    "    #배치/패팅/정수인코딩/어휘집합 구축 등 다양한 기능 제공"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "0ilmXYaHFb8d",
    "outputId": "eb297503-954b-42cb-fdd2-72d1f29b4653"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<torchtext.legacy.data.iterator.BucketIterator at 0x7f628f4ea450>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_iterator"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "s10xfbJEGWEo"
   },
   "source": [
    "## 모델 구축"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "j8gOlTDrGX7T"
   },
   "source": [
    "#### 1. Encoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "id": "ATdlSCc8GMIe"
   },
   "outputs": [],
   "source": [
    "class Encoder(nn.Module):\n",
    "    #input dim: 입력 벡터의 차원\n",
    "    #emb_dim; 임베딩 벡터의 차원\n",
    "    #hid_dim: 은닉상태 벡터의 차원\n",
    "    #n_layers: layer의 개수 <<, 논문에서는 LSTM을 Encoder, Decoder각각 4개씩 쌓았음\n",
    "    def __init__(self, input_dim, emb_dim, hid_dim, n_layers, dropout):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.hid_dim = hid_dim\n",
    "        self.n_layers = n_layers\n",
    "        \n",
    "        self.embedding = nn.Embedding(input_dim, emb_dim)\n",
    "        #임베딩 층\n",
    "        \n",
    "        self.rnn = nn.LSTM(emb_dim, hid_dim, n_layers, dropout = dropout)\n",
    "        #LSTM사용 / n_layer만큼 쌓는다\n",
    "\n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "        \n",
    "    def forward(self, src):\n",
    "        \n",
    "        #src = [src len, batch size] << 이런 식의 shape\n",
    "        \n",
    "        embedded = self.dropout(self.embedding(src)) \n",
    "        \n",
    "        #embedded = [src len, batch size, emb dim]\n",
    "        \n",
    "        outputs, (hidden, cell) = self.rnn(embedded)\n",
    "        #임베딩 된\n",
    "        \n",
    "        #outputs = [src len, batch size, hid dim * n directions]\n",
    "        #hidden = [n layers * n directions, batch size, hid dim]\n",
    "        #cell = [n layers * n directions, batch size, hid dim]\n",
    "        \n",
    "        #outputs are always from the top hidden layer\n",
    "        \n",
    "        return hidden, cell"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "iwSJ1Ya4HWWF"
   },
   "source": [
    "### 2. Decoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "id": "94TQyKlAHnwm"
   },
   "outputs": [],
   "source": [
    "class Decoder(nn.Module):\n",
    "    def __init__(self, output_dim, emb_dim, hid_dim, n_layers, dropout):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.output_dim = output_dim\n",
    "        self.hid_dim = hid_dim\n",
    "        self.n_layers = n_layers\n",
    "        \n",
    "        self.embedding = nn.Embedding(output_dim, emb_dim)\n",
    "        \n",
    "        self.rnn = nn.LSTM(emb_dim, hid_dim, n_layers, dropout = dropout)\n",
    "        \n",
    "        self.fc_out = nn.Linear(hid_dim, output_dim)\n",
    "        \n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "        \n",
    "    def forward(self, input, hidden, cell):\n",
    "        \n",
    "        #input = [batch size]\n",
    "        #hidden = [n layers * n directions, batch size, hid dim]\n",
    "        #cell = [n layers * n directions, batch size, hid dim]\n",
    "        \n",
    "        #n directions in the decoder will both always be 1, therefore:\n",
    "        #hidden = [n layers, batch size, hid dim]\n",
    "        #context = [n layers, batch size, hid dim]\n",
    "        \n",
    "        input = input.unsqueeze(0)\n",
    "        \n",
    "        #input = [1, batch size]\n",
    "        \n",
    "        embedded = self.dropout(self.embedding(input))\n",
    "        \n",
    "        #embedded = [1, batch size, emb dim]\n",
    "                \n",
    "        output, (hidden, cell) = self.rnn(embedded, (hidden, cell))\n",
    "        \n",
    "        #output = [seq len, batch size, hid dim * n directions]\n",
    "        #hidden = [n layers * n directions, batch size, hid dim]\n",
    "        #cell = [n layers * n directions, batch size, hid dim]\n",
    "        \n",
    "        #seq len and n directions will always be 1 in the decoder, therefore:\n",
    "        #output = [1, batch size, hid dim]\n",
    "        #hidden = [n layers, batch size, hid dim]\n",
    "        #cell = [n layers, batch size, hid dim]\n",
    "        \n",
    "        prediction = self.fc_out(output.squeeze(0))\n",
    "        \n",
    "        #prediction = [batch size, output dim]\n",
    "        \n",
    "        return prediction, hidden, cell"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ChDKWQ7jH4v8"
   },
   "source": [
    "### 3. Seq2seq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "id": "TrY9S2NLH4Gt"
   },
   "outputs": [],
   "source": [
    "class Seq2Seq(nn.Module):\n",
    "    def __init__(self, encoder, decoder, device):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.encoder = encoder\n",
    "        self.decoder = decoder\n",
    "        self.device = device\n",
    "        \n",
    "        assert encoder.hid_dim == decoder.hid_dim, \\\n",
    "            \"Hidden dimensions of encoder and decoder must be equal!\"\n",
    "        assert encoder.n_layers == decoder.n_layers, \\\n",
    "            \"Encoder and decoder must have equal number of layers!\"\n",
    "        \n",
    "    def forward(self, src, trg, teacher_forcing_ratio = 0.5):\n",
    "        \n",
    "        #src = [src len, batch size]\n",
    "        #trg = [trg len, batch size]\n",
    "        #teacher_forcing_ratio is probability to use teacher forcing\n",
    "        #e.g. if teacher_forcing_ratio is 0.75 we use ground-truth inputs 75% of the time\n",
    "        \n",
    "        batch_size = trg.shape[1]\n",
    "        trg_len = trg.shape[0]\n",
    "        trg_vocab_size = self.decoder.output_dim\n",
    "        \n",
    "        #tensor to store decoder outputs\n",
    "        outputs = torch.zeros(trg_len, batch_size, trg_vocab_size).to(self.device)\n",
    "        \n",
    "        #last hidden state of the encoder is used as the initial hidden state of the decoder\n",
    "        hidden, cell = self.encoder(src)\n",
    "        \n",
    "        #first input to the decoder is the  tokens\n",
    "        input = trg[0,:]\n",
    "        \n",
    "        for t in range(1, trg_len):\n",
    "            \n",
    "            #insert input token embedding, previous hidden and previous cell states\n",
    "            #receive output tensor (predictions) and new hidden and cell states\n",
    "            output, hidden, cell = self.decoder(input, hidden, cell)\n",
    "            \n",
    "            #place predictions in a tensor holding predictions for each token\n",
    "            outputs[t] = output\n",
    "            \n",
    "            #decide if we are going to use teacher forcing or not\n",
    "            teacher_force = random.random() < teacher_forcing_ratio\n",
    "            \n",
    "            #get the highest predicted token from our predictions\n",
    "            top1 = output.argmax(1) \n",
    "            \n",
    "            #if teacher forcing, use actual next token as next input\n",
    "            #if not, use predicted token\n",
    "            input = trg[t] if teacher_force else top1\n",
    "        \n",
    "        return outputs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "crGkoaUsH-1o"
   },
   "source": [
    "## 훈련하기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "yjCHpe0JH9Qn"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "provenance": []
  },
  "gpuClass": "standard",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4 (tags/v3.10.4:9d38120, Mar 23 2022, 23:13:41) [MSC v.1929 64 bit (AMD64)]"
  },
  "vscode": {
   "interpreter": {
    "hash": "29dab91150d01cac236a5af542923693a098a0071198c2c5337fc09156394190"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}