2. preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "from transformers import AutoTokenizer\n",
    "from src.MyDataset import MyDataset_WithPreprocess"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# BERT Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#### BERT MODEL\n",
    "model_name = \"distilbert-base-uncased\"\n",
    "bertTokenizer = AutoTokenizer.from_pretrained(model_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "train_dataset = MyDataset_WithPreprocess(path_data=\"./data/train_unclean.csv.zip\",\n",
    "                                         emb_model=bertTokenizer,\n",
    "                                         emb_type=\"bert\",\n",
    "                                         max_seq_len=200,\n",
    "                                         input_type=\"unclean\",\n",
    "                                         store_processed=True,\n",
    "                                         output_dir=\"./data/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "val_dataset = MyDataset_WithPreprocess(path_data=\"./data/val_unclean.csv.zip\",\n",
    "                                       emb_model=bertTokenizer,\n",
    "                                       emb_type=\"bert\",\n",
    "                                       max_seq_len=200,\n",
    "                                       input_type=\"unclean\",\n",
    "                                       store_processed=True,\n",
    "                                       output_dir=\"./data/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "test_dataset = MyDataset_WithPreprocess(path_data=\"./data/test_unclean.csv.zip\",\n",
    "                                        emb_model=bertTokenizer,\n",
    "                                        emb_type=\"bert\",\n",
    "                                        max_seq_len=200,\n",
    "                                        input_type=\"unclean\",\n",
    "                                        store_processed=True,\n",
    "                                        output_dir=\"./data/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "del bertTokenizer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# GloVe Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gensim.downloader as api\n",
    "model_name = 'glove-wiki-gigaword-300' \n",
    "glove_model = api.load(model_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "train_dataset = MyDataset_WithPreprocess(path_data=\"./data/train_unclean.csv.zip\",\n",
    "                                         emb_model=glove_model,\n",
    "                                         emb_type=\"gensim\",\n",
    "                                         max_seq_len=200,\n",
    "                                         input_type=\"unclean\",\n",
    "                                         store_processed=True,\n",
    "                                         output_dir=\"./data/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "val_dataset = MyDataset_WithPreprocess(path_data=\"./data/val_unclean.csv.zip\",\n",
    "                                       emb_model=glove_model,\n",
    "                                       emb_type=\"gensim\",\n",
    "                                       max_seq_len=200,\n",
    "                                       input_type=\"unclean\",\n",
    "                                       store_processed=True,\n",
    "                                       output_dir=\"./data/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "test_dataset = MyDataset_WithPreprocess(path_data=\"./data/test_unclean.csv.zip\",\n",
    "                                        emb_model=glove_model,\n",
    "                                        emb_type=\"gensim\",\n",
    "                                        max_seq_len=200,\n",
    "                                        input_type=\"unclean\",\n",
    "                                        store_processed=True,\n",
    "                                        output_dir=\"./data/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "del glove_model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Word2Vec Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gensim.downloader as api\n",
    "model_name = 'word2vec-google-news-300' \n",
    "word2vec_model = api.load(model_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "train_dataset = MyDataset_WithPreprocess(path_data=\"./data/train_unclean.csv.zip\",\n",
    "                                         emb_model=word2vec_model,\n",
    "                                         emb_type=\"gensim\",\n",
    "                                         max_seq_len=200,\n",
    "                                         input_type=\"unclean\",\n",
    "                                         store_processed=True,\n",
    "                                         output_dir=\"./data/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "val_dataset = MyDataset_WithPreprocess(path_data=\"./data/val_unclean.csv.zip\",\n",
    "                                       emb_model=word2vec_model,\n",
    "                                       emb_type=\"gensim\",\n",
    "                                       max_seq_len=200,\n",
    "                                       input_type=\"unclean\",\n",
    "                                       store_processed=True,\n",
    "                                       output_dir=\"./data/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "test_dataset = MyDataset_WithPreprocess(path_data=\"./data/test_unclean.csv.zip\",\n",
    "                                        emb_model=word2vec_model,\n",
    "                                        emb_type=\"gensim\",\n",
    "                                        max_seq_len=200,\n",
    "                                        input_type=\"unclean\",\n",
    "                                        store_processed=True,\n",
    "                                        output_dir=\"./data/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "del word2vec_model"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}