LSTM-CNN_concat.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch \n",
    "import os, copy\n",
    "from collections import OrderedDict\n",
    "\n",
    "from utils.sample_parameters import ParamSamples\n",
    "from utils.custom_loss import opportunity_loss\n",
    "from utils.train_n_test import TrainTest\n",
    "from models.LSTM_CNN_concat import LSTM_CNN_concat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "balance_params = {\n",
    "    'START': 1000, \n",
    "    'REWARD': 10, \n",
    "    'FINE': 100, \n",
    "    'NORM_HYPERPARAMS': (0,87000),\n",
    "    'NORM': 'minmax',\n",
    "    'WARM': 1\n",
    "}\n",
    "\n",
    "params_dict = {\n",
    "    # HYPERMARAMETER RANGES\n",
    "    'BATCH_SIZE': ([48, 64, 128, 256, 512], None), \n",
    "    'EPOCHS': ([20, 40, 50], None),\n",
    "    'OPTIMIZER': ([torch.optim.RMSprop], None),\n",
    "    'LEARNING_RATE': ([1e-5, 5e-5, 1e-4, 5e-4, 1e-3], None),\n",
    "    'LOSS': ([torch.nn.MSELoss()], None),\n",
    "    'EARLY_STOPPING': ([True], None),\n",
    "    'PATIENCE': (list(range(0,3)), None),\n",
    "    'MIN_DELTA': ([1e-6, 5e-6, 1e-5, 5e-5, 1e-4, 5e-4], None),\n",
    "\n",
    "    # NETWORK PARAMETER RANGES\n",
    "    'DROPOUT': (np.arange(8)/10, None),\n",
    "    ## CNN \n",
    "    'CNN_HIDDEN_DEPTH': (np.arange(2, 5), None),\n",
    "    'CNN_HIDDEN_SIZES': ([64, 128, 256, 512], \n",
    "                       {'max_depth': 'CNN_HIDDEN_DEPTH', # sample a list of maximum HIDDEN_DEPTH length \n",
    "                        'consecutive': 1}), # list must be increasing \n",
    "    'KERNEL_SIZES': ([3,5,7,9], \n",
    "                        {'max_depth': 'CNN_HIDDEN_DEPTH', \n",
    "                        'monotonic': -1}),  \n",
    "    'MAXPOOL': ([2], None),\n",
    "    \n",
    "    ## LSTM\n",
    "    'LSTM_HIDDEN_DIM': ([64, 128, 256], None),\n",
    "    'NUM_LAYERS':([1],None),\n",
    "    'BIDIRECTIONAL': ([True, False], None),\n",
    "    \n",
    "    ## FC\n",
    "    'FC_DEPTH':  (np.arange(1,3), None),\n",
    "    'FC_SIZES': ([256, 512], \n",
    "                    {'max_depth': 'FC_DEPTH',\n",
    "                    'consecutive': -1}),\n",
    "    'BALANCE': ([balance_params], None)\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "fixed_params =[{'BATCH_SIZE': 256,\n",
    "  'EPOCHS': 20,\n",
    "  'OPTIMIZER': torch.optim.RMSprop,\n",
    "  'LEARNING_RATE': 0.001,\n",
    "  'LOSS': torch.nn.MSELoss(),\n",
    "  'EARLY_STOPPING': True,\n",
    "  'PATIENCE': 0,\n",
    "  'MIN_DELTA': 5e-06,\n",
    "  'DROPOUT': 0.1,\n",
    "  'CNN_HIDDEN_DEPTH': 4,\n",
    "  'CNN_HIDDEN_SIZES': np.array([ 64, 128, 256, 512]),\n",
    "  'KERNEL_SIZES': np.array([9,7, 5, 3]),\n",
    "  'MAXPOOL': 2,\n",
    "  'LSTM_HIDDEN_DIM': 128,\n",
    "  'NUM_LAYERS': 1,\n",
    "  'BIDIRECTIONAL': True,\n",
    "  'FC_DEPTH': 1,\n",
    "  'FC_SIZES': np.array([512]), \n",
    "  'BALANCE': balance_params}]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "N = 5\n",
    "FIX = True\n",
    "PKL_NAMES = [i for i in os.listdir('data/tensors') if 'expt0' in i ]\n",
    "PKL_NAMES = [i for i in PKL_NAMES if 'persistence' not in i ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for pkl in PKL_NAMES: \n",
    "    PKL_NAME = pkl[:-4] \n",
    "    print(PKL_NAME)\n",
    "    train, val, test = torch.load('data/tensors/'+ PKL_NAME + '.pkl')\n",
    "    persistence = torch.load('data/tensors/'+ PKL_NAME + '_persistence.pkl')\n",
    "    \n",
    "    torch.nn.MSELoss()(persistence, test[1])\n",
    "    \n",
    "    if FIX: \n",
    "        param_samples = [copy.deepcopy(fixed_params[0]) for _ in range(N)]        \n",
    "    else: \n",
    "        params = ParamSamples(OrderedDict(params_dict))\n",
    "        param_samples = params.SampleAll(N)\n",
    "    \n",
    "    stats = []\n",
    "    predictions = []\n",
    "    for i in np.arange(N): \n",
    "        print(param_samples[i])\n",
    "        run = TrainTest(LSTM_CNN_concat, (train, val, test), param_samples[i], class_size=1)\n",
    "        run.train()\n",
    "        stats.append(run.test())\n",
    "        predictions.append(run.predictions)\n",
    "\n",
    "    for i in np.arange(N):\n",
    "        for key,val in stats[i].items():\n",
    "            param_samples[i][key] = val\n",
    "\n",
    "    torch.save(param_samples, 'results/LSTM-CNN_concat_experiment_N'+str(N)+'_'+PKL_NAME+'.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Can't find a way to make this a function (plot not showing), so putting this in every notebook \n",
    "# from matplotlib.animation import FuncAnimation\n",
    "# %matplotlib notebook\n",
    "\n",
    "# interval = 50\n",
    "# windowsize = 100\n",
    "# test_ = test[1]\n",
    "# pred = run.predictions\n",
    "\n",
    "# fig  = plt.figure()\n",
    "# ax   = plt.axes(xlim=(0, windowsize), ylim=(-2, 2))\n",
    "# ln1, = plt.plot([],[])\n",
    "# ln2, = plt.plot([],[])\n",
    "\n",
    "# def update_animation(f):\n",
    "#     r = np.arange(f, f + windowsize)\n",
    "#     lim = ax.set_xlim(f, f+ windowsize)\n",
    "#     ln1.set_data(r, test_[r])\n",
    "#     ln2.set_data(r, pred[r])\n",
    "\n",
    "# frames = np.arange(0, len(pred), interval)\n",
    "# ani = FuncAnimation(fig, update_animation, frames=frames) \n",
    "# plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "deeplearning",
   "language": "python",
   "name": "deeplearning"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}