quick_training.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Semi-Supervised Model Training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Part 1: Unsupervised NLP - Encoder Training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Import Packages (fastai for optimizing inference)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '2,3'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.text import *\n",
    "import pandas as pd\n",
    "from fastai.callbacks.tracker import *\n",
    "from keras_tqdm import TQDMNotebookCallback\n",
    "from tqdm.auto import tqdm\n",
    "tqdm.pandas()\n",
    "from fastai.distributed import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from architecture.model import *\n",
    "from architecture.helper import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#list of items to remove\n",
    "to_remove = [\"\\d+\", \"\\r\", \"\\n\",\"IMPRESSION:\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# file path\n",
    "text_file_path = '../data/reports_all.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load in data\n",
    "all_text_data = pd.read_csv(text_file_path,index_col=0);all_text_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# clean data by removing items from reports\n",
    "cleaned_all_text_data = clean_impressions(all_text_data,'report',to_remove)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# development set split\n",
    "train_set, val_set, test_set = shuffle_split_dataset(cleaned_all_text_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# convert raw data into language model understandable format\n",
    "text_data = report_slicer(TextLMDataBunch,\"../data/\",\n",
    "                                  train_df=train_set,\n",
    "                                  valid_df=val_set,\n",
    "                                  test_df=test_set,\n",
    "                                  text_cols='report')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Saving and Loading Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save model\n",
    "text_data.save('../data/reports_all.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load model\n",
    "text_data = load_data(\"../data/\", 'reports_all.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Basic Encoder Model Parameters Set Here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "vocab_size = len(text_data.vocab.itos)\n",
    "encoding_size = 400\n",
    "layer_size = 1152\n",
    "num_layers = 3\n",
    "\n",
    "\n",
    "\n",
    "#stable, these variables don't need to be fine-tuned (described down below)\n",
    "pad_token = 1\n",
    "hidden_dropout = 0.2\n",
    "input_dropout = 0.6\n",
    "embed_dropout = 0.1\n",
    "weight_dropout = 0.5\n",
    "output_dropout = 0.1\n",
    "qrnn_cells = False\n",
    "bidirectional = False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id=\"build_encoder\"></a>\n",
    "#### Build Encoder "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoder = us_ccds_encoder(vocab_sz=vocab_size,\n",
    "                          enc_sz=encoding_size, \n",
    "                          n_hid=layer_size, \n",
    "                          n_layers=num_layers, \n",
    "                          pad_token=pad_token, \n",
    "                          hidden_p=hidden_dropout,\n",
    "                          input_p=input_dropout, \n",
    "                          embed_p=embed_dropout, \n",
    "                          weight_p=weight_dropout, \n",
    "                          qrnn=qrnn_cells, \n",
    "                          bidir=bidirectional)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id=\"build_decoder\"></a>\n",
    "#### Build Decoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "decoder = us_ccds_decoder(n_out=vocab_size, \n",
    "                          n_hid=encoding_size, \n",
    "                          output_p=output_dropout, \n",
    "                          tie_encoder=encoder.encoder, \n",
    "                          bias=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Combine encoder + decoder into Unsupervised-Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "custom_model = SequentialRNN(encoder,decoder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn_awd = LanguageLearner(text_data, custom_model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Convert to run on GPU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "learn_awd.model = custom_model.cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "learn_awd.to_parallel()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Change Loss Function if Needed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#learn_awd.loss_func=MSELossFlat()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Find & Plot Learning Rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn_awd.lr_find()\n",
    "learn_awd.recorder.plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Start Training Process\n",
    "- **cyc_len** = epochs/iterations\n",
    "- **max_lr** = learning rate\n",
    "- **callbacks** = saved model with lowest validation loss during training session as 'bestmodel'\n",
    "- **to load best model parameters:** `{model_name}.load('bestmodel')`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "learn_awd.fit_one_cycle(cyc_len=20,max_lr=1e-3,callbacks=[SaveModelCallback(learn_awd, every='improvement', monitor='valid_loss')])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Save/load parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn_awd.save('language_modelv1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn_awd.load('language_modelv1')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Run predict on series of text to explore model language understanding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "learn_awd.predict(\"No evidence of a particular\",100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn_awd.model[0].hidden_dps"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Part 2: Supervised NLP - Decoder Training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load and clean data [using same methods](#proc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_file_path = '../data/abdoAA_500.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_labeled_data = pd.read_csv(data_file_path);all_labeled_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cleaned_all_labeled_data = clean_impressions(all_labeled_data,'IMPRESSION',to_remove)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "labeled_train_set, labeled_val_set, labeled_test_set = shuffle_split_dataset(cleaned_all_labeled_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_labeled = create_labeled_set(TextClasDataBunch,\n",
    "                             path='../data/',\n",
    "                             train_df=labeled_train_set,\n",
    "                             valid_df=labeled_val_set,\n",
    "                             test_df=labeled_test_set,\n",
    "                             text_cols='IMPRESSION',\n",
    "                             label_cols='cohort',\n",
    "                             vocab=text_data.train_ds.vocab,\n",
    "                             bs=128\n",
    "                             )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Saving and Loading Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_labeled.save('../data/AAA.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_labeled = load_data('../data','AAA.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Basic Decoder Model Parameters Set Here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_classes = 2\n",
    "decoder_layer_sizes =[50]\n",
    "decoder_dropout =[0.1]\n",
    "bptt_classifier = 70\n",
    "max_length = 2000"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Setup for Supervised Models\n",
    "<a id=\"s_de\"></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pretrained = learn_awd.model[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoder = su_ccds_encoder(bptt=bptt_classifier, \n",
    "                              max_len=max_length,\n",
    "                              custom_model=pretrained,\n",
    "                              pad_idx=pad_token)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "decoder = su_ccds_decoder(num_classes,\n",
    "                          encoding_sz=encoding_size,\n",
    "                          lin_ftrs=decoder_layer_sizes,\n",
    "                          ps=decoder_dropout)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### `text_classifier` is the function that creates the combined supervised model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = text_classifier(data_labeled, encoder, decoder)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Freeze the encoder model (`learn.unfreeze()` allows for fine-tuning of encoder model, too)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.freeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn_awd.lr_find()\n",
    "learn_awd.recorder.plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Start Training Process"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.fit_one_cycle(cyc_len=50,max_lr=1e-3,callbacks=[SaveModelCallback(learn_awd, every='improvement', monitor='valid_loss')])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save('AAA')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Predict on string easily with `{model_name}.predict({string})`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.predict('Pizza time')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.predict('Stable postsurgical appearance of graft replacement of the ascending aorta, arch, and descending thoracic aorta. Minimal decrease in size of fluid collection surrounding the descending thoracic graft and adjacent small pleural effusion.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "learn.predict('No evidence of abdominal aortic aneurysm.')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Good command if GPU util blows up"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.cuda.empty_cache()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}