GRU_trainer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 459,
   "id": "1a5df506",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 학습 파일 shuffle\n",
    "# import random\n",
    "# with open('./train_hate_dataset_v1.txt', 'r', encoding='utf-8') as f:\n",
    "#     file = f.read()\n",
    "# files = file.split('\\n')\n",
    "# with open('./test_hate_dataset_v1.txt', 'r', encoding='utf-8') as f:\n",
    "#     file2 = f.read()\n",
    "# files2 = file2.split('\\n')\n",
    "# files.extend(files2)\n",
    "# random.shuffle(files)\n",
    "# sentences = [x.split('\\t') for x in files]\n",
    "# len_sentences = len(sentences)\n",
    "# splited_len = round(len_sentences*0.9)\n",
    "# train_sentences = sentences[:splited_len]\n",
    "# test_sentences = sentences[splited_len:]\n",
    "# train_sentences = pd.DataFrame(train_sentences)\n",
    "# test_sentences = pd.DataFrame(test_sentences)\n",
    "# train_sentences.to_csv('./train_hate_dataset_v2.txt', sep='\\t', index=None, header=None)\n",
    "# test_sentences.to_csv('./test_hate_dataset_v2.txt', sep='\\t', index=None, header=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "41bd6b4f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import Dataset, DataLoader\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "import jamotools #자모 단위 토큰화\n",
    "import re\n",
    "import gluonnlp as nlp\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from scipy import stats\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import recall_score\n",
    "from sklearn.metrics import f1_score\n",
    "from gluonnlp.data import SentencepieceTokenizer\n",
    "from kobert.utils import get_tokenizer\n",
    "\n",
    "class CharDataset(Dataset):\n",
    "    def __init__(self, dataset, sent_idx, label_idx, mode):\n",
    "        self.sentences = [line[sent_idx] for line in dataset]\n",
    "        self.labels = [int(line[label_idx]) for line in dataset]\n",
    "        self.korean = re.compile('[^1!ㄱ-ㅣ가-힣]+')\n",
    "        self.mode = mode\n",
    "        tok_path = '.cache/kobert_news_wiki_ko_cased-1087f8699e.spiece'\n",
    "        self.sp = SentencepieceTokenizer(tok_path)\n",
    "        self.vocab = self.make_vocab()\n",
    "        self.vocab_size = len(self.vocab)\n",
    "        self.q3 = self.get_q3()\n",
    "        self.char2idx = {u:i for i, u in enumerate(self.vocab)}\n",
    "        self.idx2char = {i:u for i, u in enumerate(self.vocab)}\n",
    "        self.max_len = self.find_max_len()\n",
    "        \n",
    "    def __getitem__(self, i):\n",
    "        return (self.preprocess_sentence(self.sentences[i]), torch.tensor(self.labels[i]).to(torch.float32))\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.labels)\n",
    "    \n",
    "    def make_vocab(self):\n",
    "        vocab = ''\n",
    "        for sentence in self.sentences:\n",
    "            vocab+=' '+sentence\n",
    "        vocab = self.make_token(vocab)\n",
    "        vocab = set(vocab)\n",
    "        vocab = sorted(vocab)\n",
    "        vocab.append('<UNK>') #######\n",
    "        vocab.append('<PAD>')\n",
    "        return vocab\n",
    "    \n",
    "    def make_token(self, sentence):\n",
    "        if self.mode == 'jamo':\n",
    "            chars = self.korean.sub('', jamotools.split_syllables(sentence))\n",
    "            return list(chars)\n",
    "        elif self.mode == 'char':\n",
    "            chars = self.korean.sub('', sentence)\n",
    "            return list(chars)\n",
    "        elif self.mode == 'sentencepiece':\n",
    "            return self.sp(sentence)\n",
    "    \n",
    "    def preprocess_sentence(self, sentence):\n",
    "        chars = self.make_token(sentence)\n",
    "        if len(chars) < self.q3:\n",
    "            need_pad = self.q3 - len(chars)\n",
    "            chars.extend(['<PAD>']*need_pad)\n",
    "        else:\n",
    "            chars = chars[:self.q3]\n",
    "        chars = torch.tensor([self.char2idx[x] for x in chars]).to(torch.int64)\n",
    "        return chars\n",
    "    \n",
    "    def find_max_len(self):\n",
    "        return max(len(self.make_token(item)) for item in self.sentences)\n",
    "    \n",
    "    def find_max_idx(self):\n",
    "        return self.sentences[np.argmax([len(self.make_token(item)) for item in self.sentences])]\n",
    "\n",
    "    \n",
    "    def get_q3(self):\n",
    "        values = np.array([len(self.make_token(x)) for x in self.sentences])\n",
    "        return int(np.quantile(values, 0.75))\n",
    "    \n",
    "    \n",
    "    def plot_len(self):\n",
    "        values = np.array([len(self.make_token(x)) for x in self.sentences])\n",
    "        plt.hist(values, density=True, bins=80)\n",
    "        plt.ylabel('count')\n",
    "        plt.xlabel('length of sequence')\n",
    "        plt.show()\n",
    "        print('문장 최대 길이 :',self.max_len)\n",
    "        results = stats.describe(values)\n",
    "        print('min={}, max={}, mean={}, Q2={} Q3={}'.format(results[1][0], results[1][1], results[2],\n",
    "                                                          np.median(values), np.quantile(values, 0.75)))\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "82130e71",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Net(nn.Module):\n",
    "    def __init__(self, vocab_size, seq_len, embedding_dim, hidden_size, n_layers):\n",
    "        super().__init__()\n",
    "        self.dropout_prob = 0.5\n",
    "        self.embedding_dim = embedding_dim\n",
    "        self.bidirectional = 0\n",
    "        self.n_layers = n_layers\n",
    "        self.hidden_size = hidden_size\n",
    "        self.embedding = nn.Embedding(vocab_size, self.embedding_dim)\n",
    "        self.rnn = nn.GRU(self.embedding_dim, self.hidden_size, num_layers=self.n_layers, dropout=self.dropout_prob, batch_first=True)\n",
    "        self.classifier = nn.Sequential(\n",
    "            nn.Linear(self.hidden_size, 1),\n",
    "            #nn.Linear(hidden_size*(self.bidirectional+1)*self.n_layers, 1),\n",
    "            nn.Sigmoid()\n",
    "        )\n",
    "        \n",
    "    \n",
    "    def forward(self, inp):\n",
    "        x = self.embedding(inp)\n",
    "        h_0 = self._init_state(batch_size=x.size(0))\n",
    "        x, _ = self.rnn(x, h_0)\n",
    "        h_t = x[:,-1,:]\n",
    "        output = self.classifier(h_t)\n",
    "        return output\n",
    "    \n",
    "    def _init_state(self, batch_size=1):\n",
    "        weight = next(self.parameters()).data\n",
    "        return weight.new(self.n_layers, batch_size, self.hidden_size).zero_()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "id": "f0570c5b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([-1.3933e+24,  3.0774e-41])"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = torch.Tensor(2)\n",
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "id": "600cc96f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([-1.3933e+24,  3.0774e-41])"
      ]
     },
     "execution_count": 145,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a.float()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "id": "1a119a52",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================\n",
      "mode : sentencepiece\n",
      "n_layers : 2\n",
      "lr : 5e-05\n",
      "embedding_dim : 750\n",
      "hidden_size : 100\n",
      "================\n",
      "model download\n",
      "epoch 1 train accuracy 0.679 f1_score 0.383 precision 0.598 recall 0.282\n",
      "epoch 1 test accuracy 0.377 f1_score 0.484 precision 0.348 recall 0.795\n",
      "epoch 2 train accuracy 0.782 f1_score 0.665 precision 0.731 recall 0.610\n",
      "epoch 2 test accuracy 0.591 f1_score 0.436 precision 0.441 recall 0.431\n",
      "epoch 3 train accuracy 0.867 f1_score 0.806 precision 0.835 recall 0.779\n",
      "epoch 3 test accuracy 0.562 f1_score 0.465 precision 0.421 recall 0.519\n",
      "epoch 4 train accuracy 0.910 f1_score 0.870 precision 0.891 recall 0.850\n",
      "epoch 4 test accuracy 0.559 f1_score 0.477 precision 0.422 recall 0.548\n",
      "epoch 5 train accuracy 0.944 f1_score 0.920 precision 0.932 recall 0.908\n",
      "epoch 5 test accuracy 0.569 f1_score 0.491 precision 0.434 recall 0.566\n",
      "epoch 6 train accuracy 0.963 f1_score 0.948 precision 0.957 recall 0.939\n",
      "epoch 6 test accuracy 0.559 f1_score 0.496 precision 0.427 recall 0.590\n",
      "epoch 7 train accuracy 0.971 f1_score 0.959 precision 0.967 recall 0.950\n",
      "epoch 7 test accuracy 0.519 f1_score 0.516 precision 0.409 recall 0.699\n",
      "epoch 8 train accuracy 0.982 f1_score 0.975 precision 0.979 recall 0.970\n",
      "epoch 8 test accuracy 0.422 f1_score 0.499 precision 0.366 recall 0.785\n",
      "epoch 9 train accuracy 0.984 f1_score 0.977 precision 0.979 recall 0.975\n",
      "epoch 9 test accuracy 0.459 f1_score 0.498 precision 0.378 recall 0.731\n",
      "epoch 10 train accuracy 0.989 f1_score 0.985 precision 0.986 recall 0.984\n",
      "epoch 10 test accuracy 0.463 f1_score 0.498 precision 0.379 recall 0.726\n",
      "epoch 11 train accuracy 0.988 f1_score 0.983 precision 0.983 recall 0.983\n",
      "epoch 11 test accuracy 0.447 f1_score 0.492 precision 0.371 recall 0.729\n",
      "epoch 12 train accuracy 0.991 f1_score 0.987 precision 0.988 recall 0.986\n",
      "epoch 12 test accuracy 0.501 f1_score 0.497 precision 0.394 recall 0.670\n",
      "epoch 13 train accuracy 0.989 f1_score 0.985 precision 0.983 recall 0.986\n",
      "epoch 13 test accuracy 0.439 f1_score 0.495 precision 0.370 recall 0.747\n",
      "epoch 14 train accuracy 0.991 f1_score 0.988 precision 0.986 recall 0.989\n",
      "epoch 14 test accuracy 0.458 f1_score 0.494 precision 0.376 recall 0.721\n",
      "epoch 15 train accuracy 0.994 f1_score 0.991 precision 0.990 recall 0.992\n",
      "epoch 15 test accuracy 0.495 f1_score 0.497 precision 0.392 recall 0.678\n",
      "Finished Training\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "\n",
    "prefix = '../'\n",
    "epochs = 15\n",
    "batch_size = 128\n",
    "learning_rates = [5e-5]\n",
    "embedding_dim = [750]\n",
    "hidden_size = [100]\n",
    "n_layers = [2]\n",
    "\n",
    "mode = ['sentencepiece'] # jamo : 자음,모음 단위로 토큰화. char : 한글자 단위로 토큰화\n",
    "\n",
    "torch.manual_seed(21)\n",
    "np.random.seed(21)\n",
    "random.seed(21)\n",
    "torch.backends.cudnn.deterministic = True\n",
    "torch.backends.cudnn.benchmark = False\n",
    "\n",
    "device = torch.device('cuda')\n",
    "\n",
    "dataset_train = nlp.data.TSVDataset('./data/train_hate_dataset_v2.txt')\n",
    "dataset_test = nlp.data.TSVDataset('./data/test_hate_dataset_v2.txt')\n",
    "\n",
    "best_f1 = 0\n",
    "for mode0 in mode:\n",
    "    for n_layers0 in n_layers:\n",
    "        for learning_rate in learning_rates:\n",
    "            for embedding_dim0 in embedding_dim:\n",
    "                for hidden_size0 in hidden_size:\n",
    "                    \n",
    "                    print('================')\n",
    "                    print('mode : {}'.format(mode0))\n",
    "                    print('n_layers : {}'.format(n_layers0))\n",
    "                    print('lr : {}'.format(learning_rate))\n",
    "                    print('embedding_dim : {}'.format(embedding_dim0))\n",
    "                    print('hidden_size : {}'.format(hidden_size0))\n",
    "                    print('================')\n",
    "                    \n",
    "                    data_train = CharDataset(dataset_train, 0, 1, mode=mode0)\n",
    "                    data_test = CharDataset(dataset_test, 0, 1, mode=mode0)\n",
    "\n",
    "                    train_dataloader = torch.utils.data.DataLoader(data_train, batch_size=batch_size, drop_last=True)\n",
    "                    test_dataloader = torch.utils.data.DataLoader(data_test, batch_size=batch_size, drop_last=True)\n",
    "\n",
    "                    model = Net(data_train.vocab_size, data_train.get_q3(), embedding_dim0, hidden_size0, n_layers0)\n",
    "                    model.to(device)\n",
    "                    criterion = nn.BCELoss()\n",
    "                    #criterion = nn.CrossEntropyLoss()\n",
    "                    #optimizer = optim.Adadelta(model.parameters(), lr=learning_rate)\n",
    "                    #optimizer = optim.AdamW(model.parameters(), lr=learning_rate)\n",
    "                    #optimizer = optim.SGD(model.parameters(), lr=learning_rate, momentum=0.9)\n",
    "                    optimizer = optim.RMSprop(model.parameters(), lr=learning_rate, momentum=0.9, weight_decay=0)\n",
    "\n",
    "#                     scheduler = optim.lr_scheduler.LambdaLR(optimizer=optimizer,\n",
    "#                                                     lr_lambda=lambda epoch: 0.95 ** epoch,\n",
    "#                                                     last_epoch=-1,\n",
    "#                                                     verbose=False)\n",
    "                                       \n",
    "                    for epoch in range(epochs):\n",
    "\n",
    "                        running_loss = 0.0\n",
    "                        correct = 0\n",
    "                        y_true, y_pred = [], []\n",
    "                        model.train()\n",
    "                        for i, data in enumerate(train_dataloader, 0):\n",
    "                            inputs, labels = data\n",
    "                            inputs = inputs.to(device)\n",
    "                            labels = labels.to(device)\n",
    "                            optimizer.zero_grad()\n",
    "                            \n",
    "                            outputs = model(inputs).squeeze()\n",
    "\n",
    "                            loss = criterion(outputs, labels)\n",
    "                            running_loss = loss.item()\n",
    "                            loss.backward()\n",
    "                            optimizer.step()\n",
    "                            #scheduler.step()\n",
    "                            pred = (outputs>0.5).to(torch.float)\n",
    "                            y_pred.extend(pred)\n",
    "                            y_true.extend(labels)\n",
    "\n",
    "                        y_true_cpu = [int(x) for x in y_true]\n",
    "                        y_pred_cpu = [int(x) for x in y_pred]\n",
    "                        correct = sum([(x==y) for x,y in zip(y_pred, y_true)])\n",
    "                        precision = precision_score(y_true_cpu, y_pred_cpu)\n",
    "                        recall = recall_score(y_true_cpu, y_pred_cpu)\n",
    "                        f1= f1_score(y_true_cpu, y_pred_cpu)\n",
    "                        print(\"epoch {} train accuracy {:.3f} f1_score {:.3f} precision {:.3f} recall {:.3f}\".format(epoch+1, correct / (len(y_pred)), f1, precision, recall))\n",
    "\n",
    "\n",
    "                        model.eval()\n",
    "                        y_true, y_pred = [], []\n",
    "                        running_loss = 0.0\n",
    "                        correct = 0\n",
    "                        for i, data in enumerate(test_dataloader, 0):\n",
    "                            inputs, labels = data\n",
    "                            inputs = inputs.to(device)\n",
    "                            labels = labels.to(device)\n",
    "                            outputs = model(inputs).squeeze()\n",
    "                            pred = (outputs>0.5).to(torch.float)\n",
    "                            loss = criterion(outputs, labels)\n",
    "                            running_loss += loss.item()\n",
    "                            y_pred.extend(pred)\n",
    "                            y_true.extend(labels)\n",
    "                        y_true_cpu = [x.cpu() for x in y_true]\n",
    "                        y_pred_cpu = [x.cpu() for x in y_pred]\n",
    "                        correct = sum([(x==y) for x,y in zip(y_pred, y_true)])\n",
    "                        precision = precision_score(y_true_cpu, y_pred_cpu)\n",
    "                        recall = recall_score(y_true_cpu, y_pred_cpu)\n",
    "                        f1 = f1_score(y_true_cpu, y_pred_cpu)\n",
    "                        print(\"epoch {} test accuracy {:.3f} f1_score {:.3f} precision {:.3f} recall {:.3f}\".format(epoch+1, correct / (len(y_pred)), f1, precision, recall))\n",
    "                        if epoch == 5 and f1 < 0.3:\n",
    "                            break\n",
    "                            \n",
    "                        if f1 > best_f1:\n",
    "                            best_model = model\n",
    "                            best_f1 = f1\n",
    "                            torch.save(best_model.state_dict(), './check_point/gru_sentencepiece.pt')\n",
    "                    print('Finished Training')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "95e10588",
   "metadata": {},
   "source": [
    "## save jamo model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "9ebda690",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Net(\n",
       "  (embedding): Embedding(54, 750)\n",
       "  (rnn): GRU(750, 100, num_layers=2, batch_first=True, dropout=0.5)\n",
       "  (classifier): Sequential(\n",
       "    (0): Linear(in_features=100, out_features=1, bias=True)\n",
       "    (1): Sigmoid()\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "id": "40b7e111",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5293489861259337"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_f1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "e71b55c4",
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(best_model.state_dict(), './check_point/gru_jamo.pt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "e355ebae",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import random\n",
    "\n",
    "prefix = '../'\n",
    "epochs = 15\n",
    "batch_size = 128\n",
    "learning_rates = [5e-5]\n",
    "embedding_dim = [750]\n",
    "hidden_size = [100]\n",
    "n_layers = [2]\n",
    "\n",
    "mode = ['jamo'] # jamo : 자음,모음 단위로 토큰화. char : 한글자 단위로 토큰화\n",
    "\n",
    "torch.manual_seed(21)\n",
    "np.random.seed(21)\n",
    "random.seed(21)\n",
    "torch.backends.cudnn.deterministic = True\n",
    "torch.backends.cudnn.benchmark = False\n",
    "\n",
    "device = torch.device('cuda')\n",
    "\n",
    "dataset_train = nlp.data.TSVDataset('./data/train_hate_dataset_v2.txt')\n",
    "dataset_test = nlp.data.TSVDataset('./data/test_hate_dataset_v2.txt')\n",
    "\n",
    "data_train = CharDataset(dataset_train, 0, 1, mode=mode[0])\n",
    "data_test = CharDataset(dataset_test, 0, 1, mode=mode[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "0f363b10",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Net(\n",
       "  (embedding): Embedding(54, 750)\n",
       "  (rnn): GRU(750, 100, num_layers=2, batch_first=True, dropout=0.5)\n",
       "  (classifier): Sequential(\n",
       "    (0): Linear(in_features=100, out_features=1, bias=True)\n",
       "    (1): Sigmoid()\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = Net(data_train.vocab_size, data_train.get_q3(), 750, 100, 2)\n",
    "model.load_state_dict(torch.load('./check_point/gru_jamo.pt'))\n",
    "model.eval()\n",
    "model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "aee8c2a1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([24,  3, 35, 24, 37, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53,\n",
      "        53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53,\n",
      "        53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53,\n",
      "        53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53,\n",
      "        53, 53, 53, 53, 53, 53, 53, 53, 53])\n",
      "['ㅈ', 'ㄲ', 'ㅓ', 'ㅈ', 'ㅕ', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>']\n",
      "tensor([[24,  3, 35, 24, 37, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53,\n",
      "         53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53,\n",
      "         53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53,\n",
      "         53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53,\n",
      "         53, 53, 53, 53, 53, 53, 53, 53, 53]], device='cuda:0')\n",
      "tensor([[0.9890]], device='cuda:0')\n"
     ]
    }
   ],
   "source": [
    "sentence = 'ㅈ꺼져'\n",
    "model.eval()\n",
    "with torch.no_grad():\n",
    "    korean = re.compile('[^1!ㄱ-ㅣ가-힣]+')\n",
    "    chars = korean.sub('', sentence)\n",
    "    chars = data_train.make_token(chars)\n",
    "    if len(chars) < data_train.get_q3():\n",
    "        need_pad = data_train.get_q3() - len(chars)\n",
    "        chars.extend(['<PAD>']*need_pad)\n",
    "    else:\n",
    "        chars = chars[:data_train.q3]\n",
    "    chars = torch.tensor([data_train.char2idx[x] for x in chars]).to(torch.int64)\n",
    "    print(chars)\n",
    "    chars = chars.to(device)\n",
    "    chars = torch.unsqueeze(chars, 0)\n",
    "    prt = [data_train.idx2char[x.item()] for x in chars.squeeze()]\n",
    "    print(prt)\n",
    "    print(chars)\n",
    "    outputs = model(chars)\n",
    "    print(outputs)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "963ff6e8",
   "metadata": {},
   "source": [
    "## save char model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "03055594",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Net(\n",
       "  (embedding): Embedding(1668, 750)\n",
       "  (rnn): GRU(750, 100, num_layers=2, batch_first=True, dropout=0.5)\n",
       "  (classifier): Sequential(\n",
       "    (0): Linear(in_features=100, out_features=1, bias=True)\n",
       "    (1): Sigmoid()\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "408b623f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5225362872421696"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_f1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "d44a87a2",
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(best_model.state_dict(), './check_point/gru_char.pt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "b8d710a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import random\n",
    "\n",
    "prefix = '../'\n",
    "epochs = 15\n",
    "batch_size = 128\n",
    "learning_rates = [5e-5]\n",
    "embedding_dim = [750]\n",
    "hidden_size = [100]\n",
    "n_layers = [2]\n",
    "\n",
    "mode = ['char'] # jamo : 자음,모음 단위로 토큰화. char : 한글자 단위로 토큰화\n",
    "\n",
    "torch.manual_seed(21)\n",
    "np.random.seed(21)\n",
    "random.seed(21)\n",
    "torch.backends.cudnn.deterministic = True\n",
    "torch.backends.cudnn.benchmark = False\n",
    "\n",
    "device = torch.device('cuda')\n",
    "\n",
    "dataset_train = nlp.data.TSVDataset('./data/train_hate_dataset_v2.txt')\n",
    "dataset_test = nlp.data.TSVDataset('./data/test_hate_dataset_v2.txt')\n",
    "\n",
    "data_train = CharDataset(dataset_train, 0, 1, mode=mode[0])\n",
    "data_test = CharDataset(dataset_test, 0, 1, mode=mode[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "bbc7e82e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Net(\n",
       "  (embedding): Embedding(1668, 750)\n",
       "  (rnn): GRU(750, 100, num_layers=2, batch_first=True, dropout=0.5)\n",
       "  (classifier): Sequential(\n",
       "    (0): Linear(in_features=100, out_features=1, bias=True)\n",
       "    (1): Sigmoid()\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = Net(data_train.vocab_size, data_train.get_q3(), 750, 100, 2)\n",
    "model.load_state_dict(torch.load('./gru_char.pt'))\n",
    "model.eval()\n",
    "model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "c98ae418",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([1185,  965, 1667, 1667, 1667, 1667, 1667, 1667, 1667, 1667, 1667, 1667,\n",
      "        1667, 1667, 1667, 1667, 1667, 1667, 1667, 1667, 1667, 1667, 1667, 1667,\n",
      "        1667, 1667, 1667, 1667, 1667, 1667, 1667, 1667, 1667, 1667])\n",
      "['좋', '아', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>']\n",
      "tensor([[1185,  965, 1667, 1667, 1667, 1667, 1667, 1667, 1667, 1667, 1667, 1667,\n",
      "         1667, 1667, 1667, 1667, 1667, 1667, 1667, 1667, 1667, 1667, 1667, 1667,\n",
      "         1667, 1667, 1667, 1667, 1667, 1667, 1667, 1667, 1667, 1667]],\n",
      "       device='cuda:0')\n",
      "tensor([[0.0018]], device='cuda:0')\n"
     ]
    }
   ],
   "source": [
    "sentence = '좋아'\n",
    "model.eval()\n",
    "with torch.no_grad():\n",
    "    korean = re.compile('[^1!ㄱ-ㅣ가-힣]+')\n",
    "    chars = korean.sub('', sentence)\n",
    "    chars = data_train.make_token(chars)\n",
    "    if len(chars) < data_train.get_q3():\n",
    "        need_pad = data_train.get_q3() - len(chars)\n",
    "        chars.extend(['<PAD>']*need_pad)\n",
    "    else:\n",
    "        chars = chars[:data_train.q3]\n",
    "    chars = torch.tensor([data_train.char2idx[x] for x in chars]).to(torch.int64)\n",
    "    print(chars)\n",
    "    chars = chars.to(device)\n",
    "    chars = torch.unsqueeze(chars, 0)\n",
    "    prt = [data_train.idx2char[x.item()] for x in chars.squeeze()]\n",
    "    print(prt)\n",
    "    print(chars)\n",
    "    outputs = model(chars)\n",
    "    print(outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1127,
   "id": "e9be7613",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEGCAYAAABy53LJAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAXMUlEQVR4nO3df7BfdX3n8efLxKBFRQjRYoLetMTtRl1RIsVarUpB/BmnwhIWhVZGnK1s7bq6E2ZXahmdwrRbFyvjCAIiWwXFolmhjQqoU0cwF41CUNYrYgliiYgozgKGfe8f53Plm8u9N/eQfHNz730+Zs7ccz7nc87387knySvnx/dzUlVIkjRTj5ntBkiS5haDQ5LUi8EhSerF4JAk9WJwSJJ6WTzbDdgTDjzwwBoZGZntZkjSnHLDDTf8pKqWTSxfEMExMjLC6OjobDdDkuaUJD+crNxLVZKkXgwOSVIvBockqReDQ5LUi8EhSerF4JAk9WJwSJJ6MTgkSb0YHJKkXob6zfEkxwDnAIuAj1TVWRPW7wN8DDgMuBs4vqpuS3IUcBawBHgQeFdVXdO2+RJwEPB/226Orqq7htmPQSPrr9xh+bazXr2nPlqS9gpDC44ki4BzgaOArcCmJBuq6uaBaqcA91TVIUnWAWcDxwM/AV5bVT9K8mxgI7B8YLsTq8oxRCRpFgzzUtXhwFhV3VpVDwKXAmsn1FkLXNzmLweOTJKq+mZV/aiVbwEe385OJEmzbJjBsRy4fWB5KzueNexQp6q2A/cCSyfUeQPwjap6YKDsoiSbk7w7SSb78CSnJhlNMrpt27Zd6YckacBefXM8ybPoLl+9daD4xKp6DvDiNr1psm2r6ryqWlNVa5Yte8SowJKkR2mYwXEHcPDA8opWNmmdJIuB/ehukpNkBXAFcFJVfX98g6q6o/38BfBxuktikqQ9ZJjBsQlYlWRlkiXAOmDDhDobgJPb/LHANVVVSZ4MXAmsr6qvjldOsjjJgW3+scBrgJuG2AdJ0gRDC452z+I0uieivgN8sqq2JDkzyetatQuApUnGgHcA61v5acAhwBntXsbmJE8B9gE2Jvk2sJnujOX8YfVBkvRIQ/0eR1VdBVw1oeyMgfn7geMm2e69wHun2O1hu7ONkqR+9uqb45Kkvc+CeOf4rpj4TXFJWug845Ak9WJwSJJ6MTgkSb0YHJKkXgwOSVIvBockqReDQ5LUi8EhSerF4JAk9WJwSJJ6MTgkSb0YHJKkXgwOSVIvBockqReDQ5LUi8EhSerF4JAk9WJwSJJ6MTgkSb0YHJKkXgwOSVIvBockqReDQ5LUi8EhSerF4JAk9WJwSJJ6MTgkSb0YHJKkXgwOSVIvQw2OJMckuSXJWJL1k6zfJ8llbf31SUZa+VFJbkhyY/v58oFtDmvlY0k+kCTD7IMkaUdDC44ki4BzgVcCq4ETkqyeUO0U4J6qOgR4P3B2K/8J8Nqqeg5wMnDJwDYfAt4CrGrTMcPqgyTpkYZ5xnE4MFZVt1bVg8ClwNoJddYCF7f5y4Ejk6SqvllVP2rlW4DHt7OTg4AnVdV1VVXAx4DXD7EPkqQJhhkcy4HbB5a3trJJ61TVduBeYOmEOm8AvlFVD7T6W3eyT0nSEC2e7QZMJ8mz6C5fHf0otj0VOBXg6U9/+m5umSQtXMM847gDOHhgeUUrm7ROksXAfsDdbXkFcAVwUlV9f6D+ip3sE4CqOq+q1lTVmmXLlu1iVyRJ44YZHJuAVUlWJlkCrAM2TKizge7mN8CxwDVVVUmeDFwJrK+qr45Xrqo7gZ8nOaI9TXUS8Nkh9kGSNMHQgqPdszgN2Ah8B/hkVW1JcmaS17VqFwBLk4wB7wDGH9k9DTgEOCPJ5jY9pa37U+AjwBjwfeAfh9UHSdIjDfUeR1VdBVw1oeyMgfn7geMm2e69wHun2Oco8Ozd21JJ0kz5zXFJUi8GhySpF4NDktSLwSFJ6sXgkCT1YnBIknoxOCRJvRgckqReDA5JUi8GhySpF4NDktSLwSFJ6sXgkCT1YnBIknoxOCRJvRgckqReDA5JUi8GhySpF4NDktSLwSFJ6sXgkCT1YnBIknoxOCRJvRgckqReDA5JUi8GhySpF4NDktSLwSFJ6sXgkCT1YnBIknoxOCRJvRgckqRehhocSY5JckuSsSTrJ1m/T5LL2vrrk4y08qVJrk1yX5IPTtjmS22fm9v0lGH2QZK0o8XD2nGSRcC5wFHAVmBTkg1VdfNAtVOAe6rqkCTrgLOB44H7gXcDz27TRCdW1eiw2i5JmtowzzgOB8aq6taqehC4FFg7oc5a4OI2fzlwZJJU1S+r6p/pAkSStBcZZnAsB24fWN7ayiatU1XbgXuBpTPY90XtMtW7k2SyCklOTTKaZHTbtm39Wy9JmtRcvDl+YlU9B3hxm940WaWqOq+q1lTVmmXLlu3RBkrSfDbM4LgDOHhgeUUrm7ROksXAfsDd0+20qu5oP38BfJzukpgkaQ+ZUXAkuXomZRNsAlYlWZlkCbAO2DChzgbg5DZ/LHBNVdU07Vic5MA2/1jgNcBNM+mDJGn3mPapqiSPA34DODDJ/sD4/YQn8cj7FTuoqu1JTgM2AouAC6tqS5IzgdGq2gBcAFySZAz4KV24jH/2be1zliR5PXA08ENgYwuNRcAXgfN79ViStEt29jjuW4E/B54G3MDDwfFz4INTbPNrVXUVcNWEsjMG5u8Hjpti25EpdnvYzj5XkjQ80wZHVZ0DnJPkP1XV3+2hNkmS9mIz+gJgVf1dkt8DRga3qaqPDaldkqS91IyCI8klwG8Dm4GHWnEBBockLTAzHXJkDbB6uieeJEkLw0y/x3ET8JvDbIgkaW6Y6RnHgcDNSb4OPDBeWFWvG0qrJEl7rZkGx3uG2QhJ0twx06eqvjzshkiS5oaZPlX1C7qnqACWAI8FfllVTxpWwyRJe6eZnnE8cXy+DWO+FjhiWI2aS0bWX/nr+dvOevUstkSS9ozeo+NW5zPAK3Z/cyRJe7uZXqr6o4HFx9B9r8O380nSAjTTp6peOzC/HbiNR74GVpK0AMz0HsefDLshkqS5YaYvclqR5Iokd7Xp00lWDLtxkqS9z0xvjl9E97a+p7Xpf7cySdICM9PgWFZVF1XV9jZ9FFg2xHZJkvZSMw2Ou5O8McmiNr0RuHuYDZMk7Z1mGhxvBv498GPgTuBY4I+H1CZJ0l5spo/jngmcXFX3ACQ5APgbukCRJC0gMz3j+HfjoQFQVT8FnjecJkmS9mYzDY7HJNl/fKGdccz0bEWSNI/M9B///wF8Lcmn2vJxwPuG0yRJ0t5spt8c/1iSUeDlreiPqurm4TVLkrS3mvHlphYUhoUkLXC9h1WXJC1sBockqReDQ5LUi8EhSerF4JAk9WJwSJJ6MTgkSb0MNTiSHJPkliRjSdZPsn6fJJe19dcnGWnlS5Ncm+S+JB+csM1hSW5s23wgSYbZB0nSjoYWHEkWAecCrwRWAyckWT2h2inAPVV1CPB+4OxWfj/wbuCdk+z6Q8BbgFVtOmb3t16SNJVhnnEcDoxV1a1V9SBwKbB2Qp21wMVt/nLgyCSpql9W1T/TBcivJTkIeFJVXVdVBXwMeP0Q+yBJmmCYwbEcuH1geWsrm7ROVW0H7gWW7mSfW3eyTwCSnJpkNMnotm3bejZdkjSVeXtzvKrOq6o1VbVm2TJfjy5Ju8swg+MO4OCB5RWtbNI6SRYD+zH9u8zvaPuZbp+SpCEaZnBsAlYlWZlkCbAO2DChzgbg5DZ/LHBNu3cxqaq6E/h5kiPa01QnAZ/d/U2XJE1laG/xq6rtSU4DNgKLgAurakuSM4HRqtoAXABckmQM+ClduACQ5DbgScCSJK8Hjm5Du/8p8FHg8cA/tkmStIcM9fWvVXUVcNWEsjMG5u+ne5vgZNuOTFE+Cjx797VSktTHvL05LkkaDoNDktSLwSFJ6sXgkCT1YnBIknoxOCRJvRgckqReDA5JUi8GhySpF4NDktSLwSFJ6sXgkCT1YnBIknoxOCRJvRgckqReDA5JUi8GhySpF4NDktSLwSFJ6sXgkCT1YnBIknoxOCRJvSye7QbMJyPrr9xh+bazXj1LLZGk4fGMQ5LUi8EhSerF4JAk9WJwSJJ6MTgkSb0YHJKkXgwOSVIvBockqReDQ5LUy1CDI8kxSW5JMpZk/STr90lyWVt/fZKRgXWnt/JbkrxioPy2JDcm2ZxkdJjtlyQ90tCGHEmyCDgXOArYCmxKsqGqbh6odgpwT1UdkmQdcDZwfJLVwDrgWcDTgC8meWZVPdS2e1lV/WRYbZckTW2YZxyHA2NVdWtVPQhcCqydUGctcHGbvxw4Mkla+aVV9UBV/QAYa/uTJM2yYQbHcuD2geWtrWzSOlW1HbgXWLqTbQv4fJIbkpw61YcnOTXJaJLRbdu27VJHJEkPm4s3x3+/qp4PvBJ4W5KXTFapqs6rqjVVtWbZsmV7toWSNI8NMzjuAA4eWF7Ryiatk2QxsB9w93TbVtX4z7uAK/ASliTtUcMMjk3AqiQrkyyhu9m9YUKdDcDJbf5Y4Jqqqla+rj11tRJYBXw9yb5JngiQZF/gaOCmIfZBkjTB0J6qqqrtSU4DNgKLgAurakuSM4HRqtoAXABckmQM+ClduNDqfRK4GdgOvK2qHkryVOCK7v45i4GPV9U/DasPu8oXO0maj4b6BsCqugq4akLZGQPz9wPHTbHt+4D3TSi7FXju7m+pJGmm5uLNcUnSLDI4JEm9GBySpF4MDklSLwaHJKkXg0OS1IvBIUnqxeCQJPVicEiSejE4JEm9GBySpF4MDklSLwaHJKkXg0OS1IvBIUnqZajv49COfLGTpPnAMw5JUi8GhySpF4NDktSLwSFJ6sXgkCT1YnBIknrxcdxZ5OO5kuYizzgkSb0YHJKkXgwOSVIvBockqReDQ5LUi09V7UUGn7LyCStJeyvPOCRJvRgckqReDA5JUi9DDY4kxyS5JclYkvWTrN8nyWVt/fVJRgbWnd7Kb0nyipnuU5I0XEO7OZ5kEXAucBSwFdiUZENV3TxQ7RTgnqo6JMk64Gzg+CSrgXXAs4CnAV9M8sy2zc72OS9MHI5kZ7yZLmlPGeZTVYcDY1V1K0CSS4G1wOA/8muB97T5y4EPJkkrv7SqHgB+kGSs7Y8Z7HNB6hs0g3YWOtM97eV4W9LCM8zgWA7cPrC8FfjdqepU1fYk9wJLW/l1E7Zd3uZ3tk8AkpwKnNoW70tyy6PoA8CBwE8e5bZzQs5+RNGUfZ6kbq/1e7F5f5wnYZ8Xhl3p8zMmK5y33+OoqvOA83Z1P0lGq2rNbmjSnGGfFwb7vDAMo8/DvDl+B3DwwPKKVjZpnSSLgf2Au6fZdib7lCQN0TCDYxOwKsnKJEvobnZvmFBnA3Bymz8WuKaqqpWva09drQRWAV+f4T4lSUM0tEtV7Z7FacBGYBFwYVVtSXImMFpVG4ALgEvaze+f0gUBrd4n6W56bwfeVlUPAUy2z2H1odnly11zkH1eGOzzwrDb+5zuP/iSJM2M3xyXJPVicEiSejE4pjBfhzZJcnCSa5PcnGRLkre38gOSfCHJ99rP/Vt5knyg/R6+neT5s9uDRy/JoiTfTPK5tryyDXUz1oa+WdLKpxwKZy5J8uQklyf5bpLvJHnhfD/OSf5z+3N9U5JPJHncfDvOSS5McleSmwbKeh/XJCe3+t9LcvJknzUVg2MSA8OlvBJYDZzQhkGZD7YD/6WqVgNHAG9rfVsPXF1Vq4Cr2zJ0v4NVbToV+NCeb/Ju83bgOwPLZwPvr6pDgHvohsCBgaFwgPe3enPROcA/VdXvAM+l6/u8Pc5JlgN/BqypqmfTPUAzPpTRfDrOHwWOmVDW67gmOQD4C7ovUB8O/MV42MxIVTlNmIAXAhsHlk8HTp/tdg2pr5+lG/vrFuCgVnYQcEub/zBwwkD9X9ebSxPdd36uBl4OfA4I3bdpF0885nRP7b2wzS9u9TLbfejZ3/2AH0xs93w+zjw8EsUB7bh9DnjFfDzOwAhw06M9rsAJwIcHyneot7PJM47JTTZcyvIp6s5Z7dT8ecD1wFOr6s626sfAU9v8fPld/E/gvwL/ry0vBX5WVdvb8mC/dhgKBxgfCmcuWQlsAy5ql+c+kmRf5vFxrqo7gL8B/gW4k+643cD8Ps7j+h7XXTreBscCleQJwKeBP6+qnw+uq+6/IPPmOe0krwHuqqobZrste9Bi4PnAh6rqecAvefjyBTAvj/P+dIOerqQbVXtfHnlJZ97bE8fV4JjcvB7aJMlj6ULj76vqH1rxvyY5qK0/CLirlc+H38WLgNcluQ24lO5y1TnAk9tQN7Bjv6YaCmcu2Qpsrarr2/LldEEyn4/zHwI/qKptVfUr4B/ojv18Ps7j+h7XXTreBsfk5u3QJklC943971TV3w6sGhz+5WS6ex/j5Se1pzOOAO4dOCWeE6rq9KpaUVUjdMfymqo6EbiWbqgbeGSfJxsKZ86oqh8Dtyf5N63oSLqRGObtcaa7RHVEkt9of87H+zxvj/OAvsd1I3B0kv3bmdrRrWxmZvsmz946Aa8C/g/wfeC/zXZ7dmO/fp/uNPbbwOY2vYru2u7VwPeALwIHtPqhe8Ls+8CNdE+szHo/dqH/LwU+1+Z/i24MtDHgU8A+rfxxbXmsrf+t2W73o+zrocBoO9afAfaf78cZ+Evgu8BNwCXAPvPtOAOfoLuH8yu6M8tTHs1xBd7c+j4G/EmfNjjkiCSpFy9VSZJ6MTgkSb0YHJKkXgwOSVIvBockqReDQ3NekvuGsM9Dk7xqYPk9Sd65C/s7ro1Qe+3uaaE0ewwOaXKH0n2/ZXc5BXhLVb1sN+5TmhUGh+aVJO9Ksqm9e+AvW9lI+9/++e1dDZ9P8vi27gWt7uYkf93e47AEOBM4vpUf33a/OsmXktya5M+m+PwTktzY9nN2KzuD7ouXFyT56wn1D0rylfY5NyV5cSs/OsnXknwjyafa2GLj74n5biv/QB5+t8gOZ0RtXyNt/o1Jvt4+48PttQEkuS/J+5J8K8l1SZ7ayp+a5IpW/q0kvzfdfrTwGByaN5IcTffegcPpzhgOS/KStnoVcG5VPQv4GfCGVn4R8NaqOhR4CKCqHgTOAC6rqkOr6rJW93fohukef3/BYyd8/tPo3unw8vb5L0jy+qo6k+4b3CdW1bsmNPs/0A3zfSjdOzM2JzkQ+O/AH1bV89u270jyOOB84LXAYcBvzuB38m+B44EXDfTxxLZ6X+C6qnou8BXgLa38A8CXW/nzgS072Y8WmMU7ryLNGUe36Ztt+Ql0gfEvdIPfbW7lNwAjSZ4MPLGqvtbKPw68Zpr9X1lVDwAPJLmLbujqrQPrXwB8qaq2AST5e+AldMN9TGUTcGELoc9U1eYkf0D3ArGvdkMusQT4Gl1w/aCqvtf2/7/oXs4znSPpQmZT29fjeXgAvAfp3lkB3e/kqDb/cuAkgKp6CLg3yZum2Y8WGIND80mAv6qqD+9Q2F2yeWCg6CG6f/j6mriPXf77U1VfaWdFrwY+muRv6d5S94WqOmGwbpJDp9nVdna8gvC48c2Ai6vq9Em2+VU9PObQzvoz3X60wHipSvPJRuDNA/cDlid5ylSVq+pnwC+S/G4rWjew+hfAE3t+/teBP0hyYLv+fwLw5ek2SPIM4F+r6nzgI3SXhq4DXpTkkFZn3yTPpBu8byTJb7fNB4PltrYt6d4rvbKVXw0cO/57SPdu6mfspB9XA/+x1V+UZL9HuR/NUwaH5o2q+jzd5aavJbmR7h0UO/vH/xTg/CSb6a7539vKr6W7GT54c3xnn38n3cuSrgW+BdxQVZ+dfiteCnwryTfp7iGc0y51/THwiSTfpl2mqqr76S5NXZnkG+x4qejTwAFJtgCn0Y3sTFXdTHe/5PNtX1+ge3XodN4OvKz9Dm8AVj/K/WiecnRcLWhJnlBV97X59XTvbX77LDdrRpK8FHhnVU13X0ba7bzHoYXu1UlOp/u78EO6/+lLmoZnHJKkXrzHIUnqxeCQJPVicEiSejE4JEm9GBySpF7+P2YbosPo2bPUAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "문장 최대 길이 : 999\n",
      "min=0, max=999, mean=27.097073312083452, Q2=19.0 Q3=34.0\n"
     ]
    }
   ],
   "source": [
    "data_train.plot_len()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1097,
   "id": "365a2347",
   "metadata": {},
   "outputs": [],
   "source": [
    "char2idx = data_train.char2idx\n",
    "idx2char = data_train.idx2char"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "27eccd74",
   "metadata": {},
   "source": [
    "## 모델 저장"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1098,
   "id": "fd937d45",
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(model.state_dict(), './check_point/1dcnn_state.pt')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fc30e52d",
   "metadata": {},
   "source": [
    "## 모델 불러오기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1099,
   "id": "61b5a99e",
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.manual_seed(55)\n",
    "torch.cuda.manual_seed(0)\n",
    "torch.cuda.manual_seed_all(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1100,
   "id": "5cf6d171",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Net(\n",
       "  (embedding): Embedding(1668, 500)\n",
       "  (bn1): BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (conv_0): Conv1d(500, 128, kernel_size=(4,), stride=(1,))\n",
       "  (conv_1): Conv1d(500, 128, kernel_size=(5,), stride=(1,))\n",
       "  (conv_2): Conv1d(500, 128, kernel_size=(6,), stride=(1,))\n",
       "  (dropout): Dropout(p=0.2, inplace=False)\n",
       "  (bn2): BatchNorm1d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (fc): Linear(in_features=384, out_features=1, bias=True)\n",
       "  (sigmoid): Sigmoid()\n",
       ")"
      ]
     },
     "execution_count": 1100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = Net(data_train.vocab_size, data_train.get_q3())\n",
    "model.load_state_dict(torch.load('./check_point/1dcnn_state.pt'))\n",
    "model.eval()\n",
    "model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1108,
   "id": "31844748",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['안', '녕', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>', '<PAD>']\n",
      "tensor([[ 967,  284, 1667, 1667, 1667, 1667, 1667, 1667, 1667, 1667, 1667, 1667,\n",
      "         1667, 1667, 1667, 1667, 1667, 1667, 1667, 1667, 1667, 1667, 1667, 1667,\n",
      "         1667, 1667, 1667, 1667, 1667, 1667, 1667, 1667, 1667, 1667]],\n",
      "       device='cuda:0')\n",
      "tensor(0.2403, device='cuda:0')\n"
     ]
    }
   ],
   "source": [
    "sentence = '안녕'\n",
    "model.eval()\n",
    "with torch.no_grad():\n",
    "    korean = re.compile('[^1!ㄱ-ㅣ가-힣]+')\n",
    "    chars = korean.sub('', sentence)\n",
    "    chars = [x for x in chars]\n",
    "    if len(chars) < 34:\n",
    "        need_pad = 34 - len(chars)\n",
    "        chars.extend(['<PAD>']*need_pad)\n",
    "    else:\n",
    "        chars = chars[:self.q3]\n",
    "    chars = torch.tensor([char2idx[x] for x in chars]).to(torch.int64).to(device)\n",
    "    chars = torch.unsqueeze(chars, 0)\n",
    "    prt = [idx2char[x.item()] for x in chars.squeeze()]\n",
    "    print(prt)\n",
    "    print(chars)\n",
    "    outputs = model(chars)\n",
    "    print(outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 561,
   "id": "f02775ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "def inference(sentence, model):\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        korean = re.compile('[^1!ㄱ-ㅣ가-힣]+')\n",
    "        chars = korean.sub('', sentence)\n",
    "        chars = [x for x in chars]\n",
    "        if len(chars) < 34:\n",
    "            need_pad = 34 - len(chars)\n",
    "            chars.extend(['<PAD>']*need_pad)\n",
    "        else:\n",
    "            chars = chars[:self.q3]\n",
    "        chars = torch.tensor([char2idx[x] for x in chars]).to(torch.int64).to(device)\n",
    "        chars = torch.unsqueeze(chars, 0)\n",
    "        prt = [idx2char[x.item()] for x in chars.squeeze()]\n",
    "        print(prt)\n",
    "        print(chars)\n",
    "        outputs = model(chars)\n",
    "        print(outputs)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}