003 Week 3 Value Function Approximation and DQN-refine code.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym\n",
    "from PIL import Image\n",
    "import numpy as np\n",
    "from scipy.misc import imresize\n",
    "import random \n",
    "import torch\n",
    "import os\n",
    "import time\n",
    "import torch.nn as nn\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('week3')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda', index=0)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from preprocessor import Preprocessor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "preprocessor = Preprocessor('pong', 210, 160, device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = gym.make(\"Pong-v0\")\n",
    "observation = env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(210, 160, 3)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "observation.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 NOOP\n",
      "1 FIRE\n",
      "2 RIGHT\n",
      "3 LEFT\n",
      "4 RIGHTFIRE\n",
      "5 LEFTFIRE\n"
     ]
    }
   ],
   "source": [
    "for action_id, name in enumerate(env.unwrapped.get_action_meanings()):\n",
    "    print(action_id, name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "NOOP, RIGHT, LEFT= 0, 2, 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAKAAAADSCAIAAABCR1ywAAACh0lEQVR4nO3bPUoDURhAUSMD2lu4CDdgaZeVWNq6GVeSzjIbcBEp0sfOJsg0EgPODLmeUwXyMw8uHw/mTVZXP3h5ffjpLS7Ias6Qz0+nr/X2/jHDSuZx+Nyc/MztzXrSNVxP+ussTuA4geOGpS483mt/szdfuvFe+5u9+a+Y4DiB4wSOEzhO4DiB4wSOEzhO4DiB4wSOW+xe9H+4/zw25/3nMRMcJ3CcwHGzPpPF/ExwnMBxw3a3X3oNTMgExwkcJ3CcwHECxwkcJ3CcwHECxw2P93dLr4EJmeA4geMEjhM4TuA4geMEjhM4TuA4geOOgbe7vafvkkxwnMBxAscd/x/s0LDKBMcJHCdwnMBxAscJHCdwnMBxAscJHCdwnMBxAscJHCdwnMBxAscJHCdwnMBxAscJHCdwnMBxAscJHCdwnMBxAscJHCdwnMBxAscJHCdwnMBxAscJHCdwnMBxAscJHCdwnMBxAscJHDcsvQBOO3xuvl/f3qzP+q4JjhM4TuA4geMEjhM4TuA4geMEjhM4TuA4geMEjhM4TuA458EX4Nwz4DETHCdwnMBxAscJHCdwnMBxAscJHCdwnMBxAscJHCdwnMBxAscJHCdwnMBxAscJHCdwnMBxAscJHCdwnMBxAscJHCdwnMBxAscJHCdwnMBxAscJHCdwnMBxAscJHCdwnMBxAscJHCdwnMBxAscJHCdwnMBxAscJHCdwnMBxAscJHCdwnMBxAscJHCdwnMBxAscJHCdwnMBxAscJHCdwnMBxAscJHCdwnMBxAscJHCdwnMBxAscJHDdsd/ul18CETHCcwHECxwkcJ3CcwHECxwkcJ3CcwHECxwkcJ3CcwHECx30Bu1ci6JJ73gAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<PIL.Image.Image image mode=RGB size=160x210 at 0x7FC8E7D2AEB8>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Image.fromarray(observation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "ob = env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.0, False, {'ale.lives': 0}, (84, 84), torch.Size([1, 4, 84, 84]))"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for i in range(4):\n",
    "    observation, reward, done, info = env.step(3)\n",
    "    frame = preprocessor.rgb2gray(observation)\n",
    "    frames = preprocessor.phi(observation)\n",
    "frames = preprocessor.phi(observation)\n",
    "# plt.imshow(frame, cmap = \"gray\")\n",
    "reward, done, info, frame.shape, frames.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from support import show_image_from_frames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATEAAADuCAYAAABRejAmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAABABJREFUeJzt3bFt40AURVHOQiXI8bII9V+BipBjq4fZCgSYWsjjC50TK3jBxwUZ0B5zzg2g6s/qAQD/Q8SANBED0kQMSBMxIE3EgDQRA9JEDEgTMSDtdOTH5/N57vv+oik8crvdtvv9PlbveEdufo0jN38oYvu+b9fr9blVPO1yuaye8Lbc/BpHbt7rJJAmYkCaiAFpIgakiRiQJmJAmogBaSIGpIkYkCZiQJqIAWkiBqSJGJAmYkCaiAFpIgakiRiQJmJAmogBaSIGpIkYkCZiQJqIAWkiBqSJGJAmYkCaiAFpIgakiRiQJmJAmogBaSIGpIkYkCZiQJqIAWkiBqSJGJAmYkCaiAFpIgakiRiQJmJAmogBaSIGpInYQmOMbYyxegakiRiQJmJAmogBaSIGpIkYkCZiQJqIAWkiBqSJGJAmYkDaafWAdzbnXD0B8jyJAWkiBqSJGPBjXvGXW0QMSBMxIE3EgDQRA9JEDEgTMSBNxIA0EQPSfDsJ/JhXfC/sSQxIEzEgTcSANBED0kQMSBMxIE3EgDQRA9JEDEgTMSBNxIA0EQPSRAxIEzEgTcSANBED0kQMSBMxIE3EgDQRA9JEDEgTMSBNxIA0EQPSxpF/ZjnG+Nq27fN1c3jg75zzY/WId+Tml/n2zR+KGMBv43USSBMxIE3EgDQRA9JEDEgTMSBNxIA0EQPSRAxIEzEgTcSANBED0kQMSBMxIE3EgDQRA9JEDEgTMSBNxIA0EQPSRAxIEzEgTcSANBED0kQMSBMxIE3EgLTTkR+fz+e57/uLpvDI7Xbb7vf7WL3jHbn5NY7c/KGI7fu+Xa/X51bxtMvlsnrC23Lzaxy5ea+TQJqIAWkiBqSJGJAmYkCaiAFpIgakiRiQJmJAmogBaSIGpIkYkCZiQJqIAWkiBqSJGJAmYkCaiAFpIgakiRiQJmJAmogBaSIGpIkYkCZiQJqIAWkiBqSJGJAmYkCaiAFpIgakiRiQJmJAmogBaSIGpIkYkCZiQJqIAWkiBqSJGJAmYkCaiAFpIgakiRiQJmJAmogBaSIGpIkYkCZiQJqIAWkiBqSJGJAmYkCaiAFpIgakiRiQJmJAmogBaSIGpIkYkCZiQJqIAWkiBqSJGJAmYkCaiAFpIgakiRiQJmJAmogBaSIGpInYQmOMbYyxegb8mFfcvIgBaSIGpIkYkCZiQJqIAWkiBqSJGJAmYkCaiAFpp9UD3tmcc/UE+FGvuHlPYkCaiAFpIgakiRiQJmJAmogBaSIGpIkYkCZiQJqIAWnjyGcAY4yvbds+XzeHB/7OOT9Wj3hHbn6Zb9/8oYgB/DZeJ4E0EQPSRAxIEzEgTcSANBED0kQMSBMxIE3EgLR/fJdaY8Z5mucAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_image_from_frames(frames)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Build model:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "from models import DQN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "from support import run_optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Test model:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Q_net = DQN([4], 3).to(device)\n",
    "# run_optimizer(Q_net, frames, device=device, lr=1, n_iter=2, is_train=[True, True, True])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Q_net = DQN([4], 3).to(device)\n",
    "# run_optimizer(Q_net, frames, device=device, lr=0.01, n_iter=2, is_train=[True, True, True])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Q_net = DQN([4], 3).to(device)\n",
    "# run_optimizer(Q_net, frames, device=device, lr=0.0001, n_iter=2, is_train=[True, True, True])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "y_hat tensor([[0.1533, 0.0180, 0.2710]], device='cuda:0', grad_fn=<AddmmBackward>).\n",
      "y_0 tensor([[0.1533]], device='cuda:0', grad_fn=<GatherBackward>).\n",
      "y_1 tensor([[0.0180]], device='cuda:0', grad_fn=<GatherBackward>)\n",
      "y_2 tensor([[0.2710]], device='cuda:0', grad_fn=<GatherBackward>)\n",
      "new y_hat tensor([[-0.4936,  0.4138,  0.6698]], device='cuda:0', grad_fn=<AddmmBackward>)\n",
      "y_hat tensor([[-0.4936,  0.4138,  0.6698]], device='cuda:0', grad_fn=<AddmmBackward>).\n",
      "y_0 tensor([[-0.4936]], device='cuda:0', grad_fn=<GatherBackward>).\n",
      "y_1 tensor([[0.4138]], device='cuda:0', grad_fn=<GatherBackward>)\n",
      "y_2 tensor([[0.6698]], device='cuda:0', grad_fn=<GatherBackward>)\n",
      "new y_hat tensor([[-0.7037,  0.6934,  0.8499]], device='cuda:0', grad_fn=<AddmmBackward>)\n"
     ]
    }
   ],
   "source": [
    "Q_net = DQN([4], 3).to(device)\n",
    "run_optimizer(Q_net, frames, device=device, lr=0.00003, n_iter=2, is_train=[True, True, True])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# good learning rate is about 0.00003"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data loader:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "from dataloader import DataLoader"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Algorithm 1: deep Q-learning with experience replay."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "from policies import Policy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.10946072477880486"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "0.989 ** 200"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "preprocessor = Preprocessor('pong', 210, 160, device)\n",
    "D = DataLoader(32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/home/vule/projects/RL_Challenge'"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "home_dir = '/home/vule/projects/RL_Challenge'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "epsilon=1\n",
    "Q = Policy(home_dir, DQN([4,84,84], 3), 3, device, epsilon=epsilon)\n",
    "Q_hat = Policy(home_dir, DQN([4,84,84], 3), 3, device, epsilon=epsilon)\n",
    "Q_hat.update_state_dict(Q)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "from support import play"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# let play\n",
    "# expected behavior: the bar will move randomly because the Q net is created randomly.\n",
    "_ = env.reset()\n",
    "game, env, win, done = play(env, Q, preprocessor, is_e_greedy=False, render=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATEAAADuCAYAAABRejAmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAABHxJREFUeJzt3MFtIksUQNHuL4fgWQ9BOCfLESFyIghmPeRQPwJr3DZQXPU56xZ6Ek+XKpBYxxgLQNV/swcA+AkRA9JEDEgTMSBNxIA0EQPSRAxIEzEgTcSAtJctD7++vo7D4XCnUfjM5XJZrtfrOnuOPbLzc2zZ+U0ROxwOy/l8/t5UfNvb29vsEXbLzs+xZeddJ4E0EQPSRAxIEzEgTcSANBED0kQMSBOxiU6n03I6nWaPAWkiBqSJGPAw97h9iBiQJmJAmogBaSIGpIkYkCZiQJqIAWmb/tmV23p/f589AuQ5iQFpIgakuU4CD3OPr1CcxIA0EQPSRAxIEzEgTcSANBED0kQMSBMxIE3EgDQRA9JEDEgTMSBNxIA0EQPSRAxIEzEgTcSANBED0kQMSBMxIE3EgDQRA9JEDEgTMSBNxIA0EQPSRAxIEzF4Euu6zh4hScSANBGDJzHGmD1CkojBDx2Px+V4PM4eY7eeLmK+FwC2eLqIAWwhYkCaiAFpN4/YT7/k9AsNsIWTGJAmYkCaiAFpL7MHgLqPj4/ZI+yakxiQJmJAmogBaSIGpIkYkHbzXyf9UgM8kpMYkCZiQJqIAWkiBqSJGJAmYkCaiAFpIgakiRiQJmJAmogBaSIGpIkYkCZiQJqIAWkiBqSJGJAmYkCaiAFpIgakiRiQJmJAmogBaSIGpIkYkLaOMb7+8Lr+XZblz/3G4RO/xxi/Zg+xR3Z+mi/v/KaIATwb10kgTcSANBED0kQMSBMxIE3EgDQRA9JEDEgTMSBNxIA0EQPSRAxIEzEgTcSANBED0kQMSBMxIE3EgDQRA9JEDEgTMSBNxIA0EQPSRAxIEzEgTcSANBED0l62PPz6+joOh8OdRuEzl8tluV6v6+w59sjOz7Fl5zdF7HA4LOfz+XtT8W1vb2+zR9gtOz/Hlp13nQTSRAxIEzEgTcSANBED0kQMeJjT6bScTqebvqaITXSPNxT2RsSANBED0kQMSBMxIE3EgDQRA9JEDEjb9Fc8AD/x/v5+89cUsYnu8YbC3rhOAmkiBqSJGJAmYkCaiAFpIgakiRiQJmJAmogBaSIGpIkYkCZiQJqIAWkiBqSJGJAmYkCaiAFpIgakiRiQJmJAmogBaSIGpIkYkCZiQJqIAWkiBqSJGJAmYvBk1nVd1nWdPUaGiAFpIgakPXXEHKnZozHGMsaYPUbGU0cM4F+eOmI+jSg4Ho/L8XicPcZuPXXEAP7l5hHzqQQ8kpMYkCZiQJqIAWkiBqSJGJAmYkDay+wBoO7j42P2CLvmJAak3fwk5lMJeCQnMSBNxIA0EQPSRAxIEzEgTcSANBED0kQMSBMxIE3EgDQRA9JEDEgTMSBNxIA0EQPSRAxIEzEgTcSANBED0kQMSBMxIE3EgDQRA9JEDEgTMSBNxIA0EQPSRAxIEzEgTcSANBED0kQMSFvHGF9/eF3/Lsvy537j8InfY4xfs4fYIzs/zZd3flPEAJ6N6ySQJmJAmogBaSIGpIkYkCZiQJqIAWkiBqSJGJD2P5LPg6EyS5lHAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_image_from_frames(game[46][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.1487, 0.1408, 0.1439]], device='cuda:0', grad_fn=<AddmmBackward>)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Q.net(game[53][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# let play the second round:\n",
    "game, env, win, done = play(env, Q, preprocessor, is_e_greedy=False, render=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATEAAADuCAYAAABRejAmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAABJBJREFUeJzt3E1O41gUgNHnFmIFMC7vAfaEWFGUPSV7oMawAiauQQ9p6LiKxPUp5wzhKVwply92+JmWZRkAVf9sPQDAnxAxIE3EgDQRA9JEDEgTMSBNxIA0EQPSRAxIu1lz+O7ubpnn+Uyj8JmXl5fx9vY2bT3HNbLz21iz86siNs/zOBwOvzcVv+3x8XHrEa6Wnd/Gmp13OwmkiRiQJmLwh3a73djtdh8+Pk3exryEb4/YZ08owDlc5ErMKxLXyP/quwy3k0DaRSLmFQk4F1diQJqIAWkiBqSJGJAmYkDaqj8ABz56fn7eeoSr9u0R84QCl+R2EkgTMSBNxIA0EQPSRAxIEzEgTcSANBED0kQMSBMxIE3EgDQRA9JEDEgTMSBNxIA0EQPSRAxIEzEgTcSANBED0kQMSBMxIE3EgDQRA9JEDEgTMSBNxIA0EQPSRAxIEzEgTcSANBED0kQMSBMxIE3EgDQRA9JEDEgTMSBNxIA0EQPSRAxIEzEgTcSANBED0kQMSBMxIE3EgDQRA9JEDEgTsQ3t9/ux3++3HgMu5hw7L2JAmogBaSIGpIkYkCZiQJqIAWkiBqSJGJB2s/UA1+zp6WnrEeCizrHzrsSANBED0kQMSBMxIE3EgDQRA9JEDEgTMSBNxIA0EQPSRAxIEzEgTcSANBED0kQMSBMxIE3EgDQRA9JEDEiblmU5/fA0vY4xfp5vHD7xY1mW+62HuEZ2fjMn7/yqiAH8bdxOAmkiBqSJGJAmYkCaiAFpIgakiRiQJmJAmogBaSIGpIkYkCZiQJqIAWkiBqSJGJAmYkCaiAFpIgakiRiQJmJAmogBaSIGpIkYkCZiQJqIAWkiBqSJGJB2s+bw3d3dMs/zl2deX1/HGGPc39//5+ePx+N4eHhY82Wv3svLy3h7e5u2nuManbLzx+NxjDHs9Tdas/OrIjbP8zgcDl+e2e12Y4wxnp+f1zw0X3h8fNx6hKt1ys5P07/fa/93jtOt2Xm3k0CaiAFpIgakiRiQtuqNfeCjZVm2HuGquRID0kQMSPv220m/HwZckisxIE3EgDQRA9JEDEgTMSBNxIA0EQPSRAxIEzEgTcSANBED0kQMSBMxIE3EgDQRA9JEDEgTMSBNxIA0EQPSRAxIEzEgTcSANBED0kQMSBMxIE3EgDQRA9JEDEgTMSBNxIA0EQPSRAxIEzEgTcSANBED0kQMSBMxIE3EgDQRA9JEDEgTMSBNxIA0EQPSRAxIEzEgTcSANBED0kQMSBMxIE3EgDQRA9JEDEgTMSBNxIA0EQPSRAxIEzEgTcSANBED0kQMSBMxIE3EgDQRA9JEbEP7/X7s9/utx4A0EQPSRAxIEzEgTcSANBED0kQMSBMx4GJub2/H7e3ttz6miAFpIgak3Ww9wDV7enraegTIcyUGpIkYkCZiQJqIAWkiBqT56SRwMe/v79/+mK7EgDQRA9JEDEgTMSBtWpbl9MPT9DrG+Hm+cfjEj2VZ7rce4hrZ+c2cvPOrIgbwt3E7CaSJGJAmYkCaiAFpIgakiRiQJmJAmogBaSIGpP0CUeOB+xD/z3wAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_image_from_frames(game[38][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# let play the third round:\n",
    "game, env, win, done = play(env, Q, preprocessor, is_e_greedy=False, render=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(True, False)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "win, done"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATEAAADuCAYAAABRejAmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAABHxJREFUeJzt3cFtIlkUQNGqESHQ6yYIckJEhMiJIOh1k0PNahZjybKxwZ+rOmddQk/i6fJ/yZLnZVkmgKp/Rg8A8B0iBqSJGJAmYkCaiAFpIgakiRiQJmJAmogBaZt7Ht5ut8tut3vSKLzner1Ot9ttHj3HGtn5Me7Z+bsittvtpsvl8rWp+LL9fj96hNWy82Pcs/Ouk0CaiAFpIgakiRiQJmJAmogBaSIGpIkYkCZiQJqIAWkiBqSJGJAmYkCaiAFpIgakiRiQJmJAmogBaSIGpIkYkCZiQJqIAWkiBqSJGJAmYkCaiAFpIgbfdDqdptPpNHqM1RIxIE3E4MXM8zzN8zx6jIyHR+yZR2tfLPDWZvQAwP8tyzJ6hJTUddKXC7yVihjAWyIGpIkYkObFPnzT8XgcPcKqOYkBaSIGpD38OuloDfwkJzEgTcSANBED0kQMSBMxIE3EgDQRA9JEDEgTMSBNxIA0EQPSRAxIEzEgTcSANBED0kQMSBMxIE3EgDQRA9JEDEgTMSBNxIA0EQPSRAxIEzEgTcSANBED0kQMSBMxIE3EgB9zPp+n8/n80M8UMSBNxIA0EQPSRGygZ7wfgLURMSBNxIA0EQPSRAxIEzEgbTN6AGA9DofDwz/TSQxIEzEgTcSANO/EBnrG+wFYGycxIE3EgDQRA9JEDEgTMSBNxIA0EQPSRAxIEzEgbV6W5fMPz/PfaZr+PG8c3vF7WZZfo4dYIzs/zKd3/q6IAbwa10kgTcSANBED0kQMSBMxIE3EgDQRA9JEDEgTMSBNxIA0EQPSRAxIEzEgTcSANBED0kQMSBMxIE3EgDQRA9JEDEgTMSBNxIA0EQPSRAxIEzEgTcSANBED0jb3PLzdbpfdbvekUXjP9XqdbrfbPHqONbLzY9yz83dFbLfbTZfL5WtT8WX7/X70CKtl58e4Z+ddJ4E0EQPSRAxIEzEgTcSANBED0kQMSBMxIE3EgDQRA9JEDEgTMSBNxIA0EQPSRAxIEzEgTcSANBED0kQMSBMxIE3EgDQRA9JEDL7pdDpNp9Np9BirJWLwAubZ/0b+qodHzK8S8JNe+iTm14m1WJZl9AhZLx0xgI+IGJD20hFzxAY+8tIRA/iIiAFpIgakbUYPAHXH43H0CKvmJAakPfwk5lcJ+ElOYkCaiAFpIgakiRiQJmJAmogBaSIGpIkYkCZiQJqIAWkiBqSJGJAmYkCaiAFpIgakiRiQJmJAmogBaSIGpIkYkCZiQJqIAWkiBqSJGJAmYkCaiAFpIgakiRiQJmJAmogBaSIGpIkYkCZiQJqIAWkiBqSJGJAmYkCaiAFpIgakiRiQJmJAmogNdD6fp/P5PHoMSBMxIE3EgDQRA9JEDEgTMSBNxIA0EQPSRAxI24weYM0Oh8PoEeBH/ffH3Y/cfScxIE3EgDQRA9JEDEgTMSBNxIC0eVmWzz88z3+nafrzvHF4x+9lWX6NHmKN7Pwwn975uyIG8GpcJ4E0EQPSRAxIEzEgTcSANBED0kQMSBMxIE3EgLR/AXsnftnvbYgZAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_image_from_frames(game[30][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from game_generator import generate_game"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.reset()\n",
    "generate_game(1, D, Q, preprocessor, env, is_e_greedy=True, verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "Q.epsilon = 0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Win 4 - 21 Loss \n",
      "Win 6 - 21 Loss \n",
      "Win 10 - 21 Loss \n",
      "Win 9 - 21 Loss \n",
      "Win 9 - 21 Loss \n",
      "Win 8 - 21 Loss \n",
      "Win 8 - 21 Loss \n",
      "Win 9 - 21 Loss \n",
      "Win 8 - 21 Loss \n",
      "Win 10 - 21 Loss \n",
      "Win 11 - 21 Loss \n",
      "Win 15 - 21 Loss \n",
      "Win 15 - 21 Loss \n",
      "Win 8 - 21 Loss \n",
      "Win 7 - 21 Loss \n",
      "Win 8 - 21 Loss \n",
      "Win 5 - 21 Loss \n",
      "Win 5 - 21 Loss \n",
      "Win 3 - 21 Loss \n",
      "Win 6 - 21 Loss \n",
      "Win 7 - 21 Loss \n",
      "Win 12 - 21 Loss \n",
      "Win 8 - 21 Loss \n",
      "Win 16 - 21 Loss \n",
      "Win 2 - 21 Loss \n",
      "Win 13 - 21 Loss \n",
      "Win 11 - 21 Loss \n",
      "Win 8 - 21 Loss \n",
      "Win 5 - 21 Loss \n",
      "Win 8 - 21 Loss \n"
     ]
    }
   ],
   "source": [
    "generate_game(30, D, Q, preprocessor, env, True, verbose=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "78377"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(D)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_N_GAMES = 1\n",
    "n_iteration = 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "Q_net = DQN([4], 3).to(device)\n",
    "optimizer = torch.optim.RMSprop(Q_net.parameters(), lr=0.00003)\n",
    "MSE = nn.MSELoss(reduction='sum')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-1.,  0.,  0.,  0.,  0.,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.]])"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch, is_mixed = D.get_sample()\n",
    "np.array([sample[2] for sample in batch]).reshape(4, 8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_gradient_batch(batch, MSE, optimizer, n=4):\n",
    "    batch_state = [sample[0] for sample in batch]\n",
    "    batch_state = torch.cat(batch_state)\n",
    "    y = torch.from_numpy(np.array([sample[2] for sample in batch])).float().to(device).view(-1, 1)\n",
    "    print(y.view(-1, 8))\n",
    "    for i in range(n):\n",
    "        y_Q = Q_net(batch_state)\n",
    "        batch_a = [[sample[1]] for sample in batch]\n",
    "        idx = torch.LongTensor(batch_a).to(device)\n",
    "        y_Q = torch.gather(y_Q, 1, idx)\n",
    "        print(y_Q.view(-1, 8))\n",
    "        loss = MSE(y_Q, y)\n",
    "        print(loss)\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        y_Q = Q_net(batch_state)\n",
    "        y_Q = torch.gather(y_Q, 1, idx)\n",
    "    print(y_Q.view(-1, 8))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[-1.,  0.,  0.,  0.,  0.,  0.,  0.,  0.],\n",
      "        [ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.],\n",
      "        [ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.],\n",
      "        [ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.]], device='cuda:0')\n",
      "tensor([[-0.0934, -0.0282,  0.2083, -0.1210,  0.1202, -0.0145, -0.0300, -0.0912],\n",
      "        [ 0.0092, -0.0752, -0.0278,  0.1017, -0.0580,  0.0760, -0.1203, -0.1071],\n",
      "        [ 0.0887, -0.0016, -0.0274,  0.1372,  0.0702,  0.0535, -0.0151,  0.0173],\n",
      "        [ 0.0320,  0.3275, -0.0544, -0.1229,  0.0370,  0.0991, -0.0162,  0.0344]],\n",
      "       device='cuda:0', grad_fn=<ViewBackward>)\n",
      "tensor(1.1314, device='cuda:0', grad_fn=<MseLossBackward>)\n",
      "tensor([[-0.7349,  0.0320, -0.3623,  0.0853, -0.2831, -0.1571, -0.0993,  0.0130],\n",
      "        [-0.1475,  0.1540, -0.0398, -0.5346,  0.4450, -0.3818,  0.3485,  0.0125],\n",
      "        [-0.7836, -0.1421,  0.0621, -0.5218, -0.5869, -0.1158, -0.1025, -0.5881],\n",
      "        [-0.4462, -0.7535, -0.1476,  0.0479, -0.0358, -0.3986,  0.0747, -0.2005]],\n",
      "       device='cuda:0', grad_fn=<ViewBackward>)\n",
      "tensor(3.7443, device='cuda:0', grad_fn=<MseLossBackward>)\n",
      "tensor([[-0.9762, -0.0522,  0.1398, -0.1648,  0.1899,  0.1391,  0.0585, -0.1995],\n",
      "        [ 0.1537, -0.2179,  0.2992,  0.1270, -0.2397,  0.1038, -0.0620,  0.2492],\n",
      "        [ 0.1966,  0.0971, -0.0462,  0.1579,  0.1055,  0.1503,  0.1961,  0.2413],\n",
      "        [ 0.3290,  0.1355,  0.1106, -0.2061, -0.1697,  0.2400, -0.1897,  0.1878]],\n",
      "       device='cuda:0', grad_fn=<ViewBackward>)\n",
      "tensor(1.0040, device='cuda:0', grad_fn=<MseLossBackward>)\n",
      "tensor([[-0.8223,  0.0247, -0.0676,  0.1207, -0.0549, -0.0173, -0.0390,  0.1418],\n",
      "        [-0.0448,  0.1778, -0.1706, -0.0468,  0.1601, -0.0449,  0.0729, -0.0515],\n",
      "        [-0.0413, -0.0192,  0.0288, -0.1074, -0.0362, -0.0066, -0.0215, -0.0340],\n",
      "        [-0.0788, -0.0597, -0.0423,  0.1405,  0.0129, -0.0624,  0.1343, -0.0580]],\n",
      "       device='cuda:0', grad_fn=<ViewBackward>)\n",
      "tensor(0.2510, device='cuda:0', grad_fn=<MseLossBackward>)\n",
      "tensor([[-1.0120e+00, -3.1361e-02,  7.4571e-03, -7.3767e-02, -3.1851e-02,\n",
      "         -1.2455e-02, -7.7112e-03, -1.0455e-01],\n",
      "        [ 5.9894e-03, -1.6400e-01,  4.0768e-02,  7.0398e-03, -1.2033e-01,\n",
      "         -1.6334e-02, -7.8756e-02, -1.4070e-02],\n",
      "        [-2.9253e-02, -1.8989e-02, -3.9437e-02,  3.6608e-02, -1.4770e-02,\n",
      "         -3.3866e-04,  9.8609e-03, -1.8316e-02],\n",
      "        [-4.2471e-02, -2.7892e-02,  2.9940e-04, -1.0447e-01, -8.8671e-02,\n",
      "         -4.8245e-03, -6.1529e-02,  1.3054e-03]], device='cuda:0',\n",
      "       grad_fn=<ViewBackward>)\n",
      "tensor(0.0985, device='cuda:0', grad_fn=<MseLossBackward>)\n",
      "tensor([[-9.2216e-01,  1.7165e-02,  9.7973e-03,  5.9031e-02,  2.3132e-02,\n",
      "          1.6636e-02, -7.1934e-03,  7.0004e-02],\n",
      "        [ 2.3879e-03,  1.1976e-01,  1.5685e-04,  2.3556e-03,  1.0371e-01,\n",
      "          1.1988e-02,  7.9627e-02,  2.7500e-02],\n",
      "        [ 3.5434e-02,  9.1961e-03,  2.8721e-02, -1.2854e-02,  1.7596e-02,\n",
      "          2.1304e-02,  1.0265e-02,  2.7850e-02],\n",
      "        [ 5.6772e-02,  2.4640e-02,  1.6678e-03,  6.7759e-02,  2.1011e-03,\n",
      "          1.6518e-02,  5.0259e-02,  5.9521e-03]], device='cuda:0',\n",
      "       grad_fn=<ViewBackward>)\n",
      "tensor(0.0633, device='cuda:0', grad_fn=<MseLossBackward>)\n",
      "tensor([[-1.0146, -0.0182, -0.0097, -0.0396, -0.0221, -0.0122, -0.0100, -0.0523],\n",
      "        [-0.0039, -0.0919, -0.0120, -0.0065, -0.0796, -0.0122, -0.0603, -0.0210],\n",
      "        [-0.0287, -0.0153, -0.0234, -0.0015, -0.0133, -0.0061, -0.0044, -0.0194],\n",
      "        [-0.0405, -0.0193, -0.0054, -0.0512, -0.0475, -0.0180, -0.0330, -0.0084]],\n",
      "       device='cuda:0', grad_fn=<ViewBackward>)\n"
     ]
    }
   ],
   "source": [
    "test_gradient_batch(batch, MSE, optimizer, n=6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-1.1943,  0.1861,  0.1399],\n",
       "        [ 0.0013,  0.0121, -0.0018]], device='cuda:0', grad_fn=<AddmmBackward>)"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch_state = [sample[0] for sample in batch[0:2]]\n",
    "batch_state = torch.cat(batch_state)\n",
    "Q_net(batch_state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-1.0345,  0.1190,  0.0956],\n",
       "        [-0.0107, -0.0013, -0.0137],\n",
       "        [-0.0266, -0.0990, -0.0021],\n",
       "        [-0.0421,  0.0113,  0.0061]], device='cuda:0', grad_fn=<AddmmBackward>)"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch_state = [sample[0] for sample in batch[0:4]]\n",
    "batch_state = torch.cat(batch_state)\n",
    "Q_net(batch_state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DQN(\n",
       "  (conv): Sequential(\n",
       "    (0): Conv2d(4, 32, kernel_size=(8, 8), stride=(4, 4))\n",
       "    (1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (2): ReLU()\n",
       "    (3): Conv2d(32, 64, kernel_size=(4, 4), stride=(2, 2))\n",
       "    (4): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (5): ReLU()\n",
       "    (6): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1))\n",
       "    (7): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (8): ReLU()\n",
       "  )\n",
       "  (fc): Sequential(\n",
       "    (0): Linear(in_features=3136, out_features=512, bias=True)\n",
       "    (1): ReLU()\n",
       "    (2): Linear(in_features=512, out_features=3, bias=True)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Problem: Inconsistent results during test using different batch size.\n",
    "# https://discuss.pytorch.org/t/solved-inconsistent-results-during-test-using-different-batch-size/2265/7\n",
    "Q_net.train(False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.0375, -0.0112, -0.0360],\n",
       "        [-0.0363, -0.0116, -0.0361]], device='cuda:0', grad_fn=<AddmmBackward>)"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch_state = [sample[0] for sample in batch[0:2]]\n",
    "batch_state = torch.cat(batch_state)\n",
    "Q_net(batch_state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.0375, -0.0112, -0.0360],\n",
       "        [-0.0363, -0.0116, -0.0361],\n",
       "        [-0.0366, -0.0112, -0.0363],\n",
       "        [-0.0362, -0.0117, -0.0367]], device='cuda:0', grad_fn=<AddmmBackward>)"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch_state = [sample[0] for sample in batch[0:4]]\n",
    "batch_state = torch.cat(batch_state)\n",
    "Q_net(batch_state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DQN(\n",
       "  (conv): Sequential(\n",
       "    (0): Conv2d(4, 32, kernel_size=(8, 8), stride=(4, 4))\n",
       "    (1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (2): ReLU()\n",
       "    (3): Conv2d(32, 64, kernel_size=(4, 4), stride=(2, 2))\n",
       "    (4): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (5): ReLU()\n",
       "    (6): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1))\n",
       "    (7): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (8): ReLU()\n",
       "  )\n",
       "  (fc): Sequential(\n",
       "    (0): Linear(in_features=3136, out_features=512, bias=True)\n",
       "    (1): ReLU()\n",
       "    (2): Linear(in_features=512, out_features=3, bias=True)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Q_net.train(True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.8981,  0.1327,  0.0731]], device='cuda:0', grad_fn=<AddmmBackward>)"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch_state = [sample[0] for sample in batch[0:1]]\n",
    "batch_state = torch.cat(batch_state)\n",
    "Q_net(batch_state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-1.1943,  0.1861,  0.1399],\n",
       "        [ 0.0013,  0.0121, -0.0018]], device='cuda:0', grad_fn=<AddmmBackward>)"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch_state = [sample[0] for sample in batch[0:2]]\n",
    "batch_state = torch.cat(batch_state)\n",
    "Q_net(batch_state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Test policy:\n",
    "epsilon=1\n",
    "# Q = Policy(DQN([4,84,84], 3), 3, device, epsilon=epsilon)\n",
    "optimizer = torch.optim.RMSprop(Q.net.parameters(), lr=0.00003)\n",
    "MSE = nn.MSELoss(reduction='sum')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_Q_gradient_batch(batch, MSE, optimizer, n=4):\n",
    "    batch_state = [sample[0] for sample in batch]\n",
    "    batch_state = torch.cat(batch_state)\n",
    "    y = torch.from_numpy(np.array([sample[2] for sample in batch])).float().to(device).view(-1, 1)\n",
    "    print(y.view(-1, 8))\n",
    "    for i in range(n):\n",
    "        y_Q = Q.net(batch_state)\n",
    "        batch_a = [[sample[1]] for sample in batch]\n",
    "        idx = torch.LongTensor(batch_a).to(device)\n",
    "        y_Q = torch.gather(y_Q, 1, idx)\n",
    "        print(y_Q.view(-1, 8))\n",
    "        loss = MSE(y_Q, y)\n",
    "        print(loss)\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        y_Q = Q.net(batch_state)\n",
    "        y_Q = torch.gather(y_Q, 1, idx)\n",
    "    print(y_Q.view(-1, 8))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[-1.,  0.,  0.,  0.,  0.,  0.,  0.,  0.],\n",
      "        [ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.],\n",
      "        [ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.],\n",
      "        [ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.]], device='cuda:0')\n",
      "tensor([[-1.0096,  0.1648,  0.0992,  0.1742, -0.0848,  0.2019,  0.1289,  0.3390],\n",
      "        [ 0.2112, -0.0118,  0.2421,  0.1490,  0.1422,  0.0166,  0.1821,  0.1446],\n",
      "        [ 0.1781,  0.2191,  0.2039,  0.2489,  0.1301,  0.0862,  0.1990, -0.1843],\n",
      "        [ 0.0813,  0.1900,  0.2068,  0.2133,  0.4855,  0.1909,  0.2005,  0.2966]],\n",
      "       device='cuda:0', grad_fn=<ViewBackward>)\n",
      "tensor(1.2595, device='cuda:0', grad_fn=<MseLossBackward>)\n",
      "tensor([[ 0.0913, -0.0113,  0.0017,  0.0013, -0.0270, -0.0407,  0.0214,  0.0175],\n",
      "        [ 0.0362, -0.0212,  0.0463,  0.0136, -0.0204, -0.0384, -0.0392, -0.0278],\n",
      "        [-0.0560, -0.0372, -0.0108,  0.0431, -0.0177, -0.0459,  0.0366, -0.0356],\n",
      "        [-0.0474, -0.0135,  0.0195, -0.0166, -0.0039,  0.0211, -0.0050,  0.0312]],\n",
      "       device='cuda:0', grad_fn=<ViewBackward>)\n",
      "tensor(1.2186, device='cuda:0', grad_fn=<MseLossBackward>)\n",
      "tensor([[-0.9290, -0.1739, -0.0408, -0.0203, -0.2278, -0.0368, -0.1529, -0.1078],\n",
      "        [ 0.0237, -0.1279,  0.0565,  0.0151, -0.0010, -0.1018,  0.0475, -0.0360],\n",
      "        [ 0.0217, -0.0386, -0.1415, -0.0172, -0.0209, -0.3161,  0.0281, -0.2248],\n",
      "        [-0.0658, -0.0103, -0.1824, -0.0642, -0.0418, -0.0139, -0.1247,  0.0436]],\n",
      "       device='cuda:0', grad_fn=<ViewBackward>)\n",
      "tensor(0.3951, device='cuda:0', grad_fn=<MseLossBackward>)\n",
      "tensor([[-0.0530,  0.0521,  0.0129,  0.0442,  0.0333,  0.0236,  0.0634,  0.0421],\n",
      "        [ 0.0788,  0.0237,  0.0747,  0.0418,  0.0554,  0.0451,  0.0389,  0.0361],\n",
      "        [ 0.0396,  0.0261,  0.0525,  0.0857,  0.0207, -0.0039,  0.0661,  0.0650],\n",
      "        [ 0.0011,  0.0254,  0.0694,  0.0326,  0.0542,  0.0508,  0.0411,  0.0882]],\n",
      "       device='cuda:0', grad_fn=<ViewBackward>)\n",
      "tensor(0.9740, device='cuda:0', grad_fn=<MseLossBackward>)\n",
      "tensor([[-0.9732,  0.0355, -0.0094,  0.0482,  0.0312,  0.0686,  0.0156,  0.0272],\n",
      "        [ 0.0841, -0.0090,  0.0677,  0.0282,  0.0482,  0.0458,  0.0575,  0.0169],\n",
      "        [ 0.0437,  0.0898,  0.0524,  0.0625,  0.0356, -0.0178,  0.0886,  0.0509],\n",
      "        [ 0.0069,  0.0229,  0.0063,  0.0342,  0.1185,  0.0496, -0.0033,  0.1096]],\n",
      "       device='cuda:0', grad_fn=<ViewBackward>)\n"
     ]
    }
   ],
   "source": [
    "test_Q_gradient_batch(batch, MSE, optimizer, n=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "# test on real case:\n",
    "# Test policy function:\n",
    "epsilon=1\n",
    "Q = Policy(home_dir, DQN([4,84,84], 3), 3, device, epsilon=epsilon)\n",
    "Q_hat = Policy(home_dir, DQN([4,84,84], 3), 3, device, epsilon=epsilon)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "Q_hat.update_state_dict(Q)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[-1.0000,  0.1612,  0.0770,  0.1793, -0.0666,  0.2369,  0.1021,  0.2963],\n",
      "        [ 0.2063, -0.0494,  0.2415,  0.1384,  0.1268,  0.1504,  0.1835,  0.1275],\n",
      "        [ 0.1816,  0.1921,  0.2081,  0.2596,  0.1067,  0.0927,  0.2145, -0.2205],\n",
      "        [ 0.1382,  0.1588,  0.2495,  0.2206,  0.4698,  0.2094,  0.1692,  0.2853]],\n",
      "       device='cuda:0')\n"
     ]
    }
   ],
   "source": [
    "y = Q.get_target(batch, is_mixed=is_mixed)\n",
    "print(y.view(-1, 8))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[-1.0000,  0.1612,  0.0770,  0.1793, -0.0666,  0.2369,  0.1021,  0.2963],\n",
      "        [ 0.2063, -0.0494,  0.2415,  0.1384,  0.1268,  0.1504,  0.1835,  0.1275],\n",
      "        [ 0.1816,  0.1921,  0.2081,  0.2596,  0.1067,  0.0927,  0.2145, -0.2205],\n",
      "        [ 0.1382,  0.1588,  0.2495,  0.2206,  0.4698,  0.2094,  0.1692,  0.2853]],\n",
      "       device='cuda:0')\n"
     ]
    }
   ],
   "source": [
    "y = Q_hat.get_target(batch, is_mixed=is_mixed)\n",
    "print(y.view(-1, 8))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "Q.min_loss = 0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Q.max_iter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.0558, device='cuda:0', grad_fn=<MseLossBackward>)\n",
      "tensor([[-1.6155,  0.1971,  0.0958,  0.2629,  0.0958,  0.3004,  0.1444,  0.0916],\n",
      "        [ 0.2432, -0.1010,  0.2090,  0.2077,  0.1422,  0.3417,  0.2714,  0.1769],\n",
      "        [ 0.2988,  0.2824,  0.2086,  0.3081,  0.0611,  0.0270,  0.3029, -0.5430],\n",
      "        [ 0.3067,  0.0882,  0.3282,  0.2632,  0.6804,  0.3716,  0.1414,  0.3219]],\n",
      "       device='cuda:0', grad_fn=<ViewBackward>)\n"
     ]
    }
   ],
   "source": [
    "Q.update_Q(y, batch, verbose=True)\n",
    "    \n",
    "print(Q.get_y_hat(batch).view(-1, 8))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "MAX_N_GAMES = 1\n",
    "n_iteration = 1\n",
    "def train_model(n_iteration, verbose=True):\n",
    "    for i in range(n_iteration):\n",
    "        if verbose:\n",
    "            print(i)\n",
    "        batch, is_mixed = D.get_sample()\n",
    "        rewards = [t[2] for t in batch]\n",
    "        if verbose:\n",
    "            print(np.array(rewards).reshape(4,8), is_mixed)\n",
    "        y = Q_hat.get_target(batch, is_mixed=is_mixed).detach()\n",
    "        if verbose:\n",
    "            print(y.view(4,8))\n",
    "        if verbose:\n",
    "            old_y = Q.get_y_hat(batch)\n",
    "            print(old_y.view(4,8))\n",
    "        Q.update_Q(y, batch, verbose)\n",
    "        if verbose:\n",
    "            new_y = Q.get_y_hat(batch)\n",
    "            print(new_y.view(4,8))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "Q.max_iter = 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "Q.min_loss = 0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "[[0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0.]] None\n",
      "tensor([[-0.0523,  0.1772,  0.1516,  0.2539,  0.1959, -0.2842,  0.0354,  0.1629],\n",
      "        [-0.0013,  0.1693,  0.1987,  0.1536,  0.1366,  0.0640,  0.1029, -0.0069],\n",
      "        [ 0.1641,  0.0879, -0.5292,  0.1647,  0.1119,  0.1476,  0.1843,  0.0955],\n",
      "        [ 0.1550,  0.1408,  0.0323,  0.2396,  0.1594,  0.1327, -0.4884,  0.1518]],\n",
      "       device='cuda:0')\n",
      "tensor([[-0.0205,  0.1091,  0.2151,  0.3429,  0.2541, -0.2837, -0.2440,  0.2415],\n",
      "        [-0.0245,  0.2409,  0.2239,  0.1542,  0.1467,  0.1455,  0.1556,  0.0425],\n",
      "        [ 0.2054,  0.0622, -0.0754,  0.1641,  0.1944,  0.1778,  0.2191,  0.1457],\n",
      "        [ 0.1883,  0.1692,  0.0304,  0.3511,  0.2028,  0.2587, -0.5210,  0.1453]],\n",
      "       device='cuda:0', grad_fn=<ViewBackward>)\n",
      "tensor(0.3765, device='cuda:0', grad_fn=<MseLossBackward>)\n",
      "tensor(1.2836, device='cuda:0', grad_fn=<MseLossBackward>)\n",
      "tensor(1.3289, device='cuda:0', grad_fn=<MseLossBackward>)\n",
      "tensor(0.1275, device='cuda:0', grad_fn=<MseLossBackward>)\n",
      "tensor([[-0.0526,  0.1973,  0.1594,  0.2505,  0.1936, -0.2404,  0.0597,  0.1786],\n",
      "        [-0.0225,  0.1723,  0.1847,  0.1053,  0.1451,  0.0711,  0.1056, -0.0117],\n",
      "        [ 0.1540,  0.1241, -0.4746,  0.1474,  0.1462,  0.1453,  0.1700,  0.1102],\n",
      "        [ 0.1259,  0.1273,  0.0551,  0.2281,  0.1516,  0.1469, -0.4405,  0.1531]],\n",
      "       device='cuda:0', grad_fn=<ViewBackward>)\n"
     ]
    }
   ],
   "source": [
    "train_model(1, verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "Win 1 - 21 Loss \n",
      "Win 5 - 21 Loss \n",
      "Win 4 - 21 Loss \n",
      "Win 4 - 21 Loss \n",
      "Win 6 - 21 Loss \n",
      "5\n",
      "Win 10 - 21 Loss \n",
      "Win 4 - 21 Loss \n",
      "Win 3 - 21 Loss \n",
      "Win 12 - 21 Loss \n",
      "Win 3 - 21 Loss \n",
      "10\n",
      "Win 6 - 21 Loss \n",
      "Win 5 - 21 Loss \n",
      "Win 10 - 21 Loss \n",
      "Win 3 - 21 Loss \n",
      "15\n",
      "Win 6 - 21 Loss \n",
      "Win 3 - 21 Loss \n",
      "Win 7 - 21 Loss \n",
      "Win 5 - 21 Loss \n",
      "Win 10 - 21 Loss \n",
      "20\n",
      "Win 3 - 21 Loss \n",
      "Win 18 - 21 Loss \n",
      "Win 2 - 21 Loss \n",
      "Win 5 - 21 Loss \n",
      "Win 8 - 21 Loss \n",
      "25\n",
      "Win 11 - 21 Loss \n",
      "Win 8 - 21 Loss \n",
      "Win 3 - 21 Loss \n",
      "Win 9 - 21 Loss \n",
      "30\n",
      "Win 7 - 21 Loss \n",
      "Win 7 - 21 Loss \n",
      "Win 2 - 21 Loss \n",
      "Win 4 - 21 Loss \n",
      "35\n",
      "Win 4 - 21 Loss \n",
      "Win 16 - 21 Loss \n",
      "Win 3 - 21 Loss \n",
      "Win 11 - 21 Loss \n",
      "Win 5 - 21 Loss \n",
      "40\n",
      "Win 3 - 21 Loss \n",
      "Win 9 - 21 Loss \n",
      "Win 12 - 21 Loss \n",
      "Win 10 - 21 Loss \n",
      "Win 9 - 21 Loss \n",
      "45\n",
      "Win 8 - 21 Loss \n",
      "Win 10 - 21 Loss \n",
      "Win 9 - 21 Loss \n",
      "Win 11 - 21 Loss \n",
      "Win 10 - 21 Loss \n",
      "50\n",
      "Win 8 - 21 Loss \n",
      "Win 6 - 21 Loss \n",
      "Win 10 - 21 Loss \n",
      "Win 11 - 21 Loss \n",
      "Win 10 - 21 Loss \n",
      "55\n",
      "Win 3 - 21 Loss \n",
      "Win 15 - 21 Loss \n",
      "Win 6 - 21 Loss \n",
      "Win 2 - 21 Loss \n",
      "Win 3 - 21 Loss \n",
      "60\n",
      "Win 7 - 21 Loss \n",
      "Win 9 - 21 Loss \n",
      "Win 9 - 21 Loss \n",
      "Win 2 - 21 Loss \n",
      "Win 7 - 21 Loss \n",
      "65\n",
      "Win 10 - 21 Loss \n",
      "Win 10 - 21 Loss \n",
      "Win 7 - 21 Loss \n",
      "Win 13 - 21 Loss \n",
      "Win 5 - 21 Loss \n",
      "70\n",
      "Win 6 - 21 Loss \n",
      "Win 8 - 21 Loss \n",
      "Win 11 - 21 Loss \n",
      "Win 7 - 21 Loss \n",
      "Win 5 - 21 Loss \n",
      "75\n",
      "Win 3 - 21 Loss \n",
      "Win 9 - 21 Loss \n",
      "Win 7 - 21 Loss \n",
      "Win 6 - 21 Loss \n",
      "Win 12 - 21 Loss \n",
      "80\n",
      "Win 7 - 21 Loss \n",
      "Win 5 - 21 Loss \n",
      "Win 6 - 21 Loss \n",
      "Win 3 - 21 Loss \n",
      "Win 6 - 21 Loss \n",
      "85\n",
      "Win 6 - 21 Loss \n",
      "Win 6 - 21 Loss \n",
      "Win 5 - 21 Loss \n",
      "Win 4 - 21 Loss \n",
      "Win 6 - 21 Loss \n",
      "90\n",
      "Win 7 - 21 Loss \n",
      "Win 4 - 21 Loss \n",
      "Win 11 - 21 Loss \n",
      "Win 9 - 21 Loss \n",
      "Win 8 - 21 Loss \n",
      "95\n",
      "Win 6 - 21 Loss \n",
      "Win 3 - 21 Loss \n",
      "Win 6 - 21 Loss \n",
      "Win 7 - 21 Loss \n",
      "Win 9 - 21 Loss \n",
      "100\n",
      "Win 2 - 21 Loss \n",
      "Win 8 - 21 Loss \n",
      "Win 4 - 21 Loss \n",
      "Win 8 - 21 Loss \n",
      "Win 4 - 21 Loss \n",
      "105\n",
      "Win 10 - 21 Loss \n",
      "Win 4 - 21 Loss \n",
      "Win 10 - 21 Loss \n",
      "Win 2 - 21 Loss \n",
      "Win 6 - 21 Loss \n",
      "110\n",
      "Win 12 - 21 Loss \n",
      "Win 10 - 21 Loss \n",
      "Win 9 - 21 Loss \n",
      "Win 11 - 21 Loss \n",
      "Win 4 - 21 Loss \n",
      "115\n",
      "Win 11 - 21 Loss \n",
      "Win 6 - 21 Loss \n",
      "Win 8 - 21 Loss \n",
      "Win 1 - 21 Loss \n",
      "Win 7 - 21 Loss \n",
      "120\n",
      "Win 8 - 21 Loss \n",
      "Win 5 - 21 Loss \n",
      "Win 8 - 21 Loss \n",
      "Win 11 - 21 Loss \n",
      "Win 8 - 21 Loss \n",
      "125\n",
      "Win 14 - 21 Loss \n",
      "Win 8 - 21 Loss \n",
      "Win 3 - 21 Loss \n",
      "Win 4 - 21 Loss \n",
      "Win 6 - 21 Loss \n",
      "130\n",
      "Win 3 - 21 Loss \n",
      "Win 7 - 21 Loss \n",
      "Win 7 - 21 Loss \n",
      "Win 6 - 21 Loss \n",
      "Win 7 - 21 Loss \n",
      "135\n",
      "Win 9 - 21 Loss \n",
      "Win 5 - 21 Loss \n",
      "Win 6 - 21 Loss \n",
      "Win 4 - 21 Loss \n",
      "Win 2 - 21 Loss \n",
      "140\n",
      "Win 7 - 21 Loss \n",
      "Win 6 - 21 Loss \n",
      "Win 7 - 21 Loss \n",
      "Win 3 - 21 Loss \n",
      "Win 6 - 21 Loss \n",
      "145\n",
      "Win 8 - 21 Loss \n",
      "Win 8 - 21 Loss \n",
      "Win 10 - 21 Loss \n",
      "Win 9 - 21 Loss \n",
      "Win 8 - 21 Loss \n",
      "150\n",
      "Win 5 - 21 Loss \n",
      "Win 9 - 21 Loss \n",
      "Win 8 - 21 Loss \n",
      "Win 10 - 21 Loss \n",
      "Win 10 - 21 Loss \n",
      "155\n",
      "Win 7 - 21 Loss \n",
      "Win 8 - 21 Loss \n",
      "Win 9 - 21 Loss \n",
      "Win 7 - 21 Loss \n",
      "Win 4 - 21 Loss \n",
      "160\n",
      "Win 4 - 21 Loss \n",
      "Win 6 - 21 Loss \n",
      "Win 3 - 21 Loss \n",
      "Win 5 - 21 Loss \n",
      "165\n",
      "Win 7 - 21 Loss \n",
      "Win 4 - 21 Loss \n",
      "Win 6 - 21 Loss \n",
      "Win 13 - 21 Loss \n",
      "Win 12 - 21 Loss \n",
      "170\n",
      "Win 3 - 21 Loss \n",
      "Win 4 - 21 Loss \n",
      "Win 4 - 21 Loss \n",
      "Win 10 - 21 Loss \n",
      "Win 9 - 21 Loss \n",
      "175\n",
      "Win 3 - 21 Loss \n",
      "Win 4 - 21 Loss \n",
      "Win 10 - 21 Loss \n",
      "Win 5 - 21 Loss \n",
      "Win 8 - 21 Loss \n",
      "180\n",
      "Win 8 - 21 Loss \n",
      "Win 3 - 21 Loss \n",
      "Win 5 - 21 Loss \n",
      "Win 8 - 21 Loss \n",
      "Win 7 - 21 Loss \n",
      "185\n",
      "Win 10 - 21 Loss \n",
      "Win 4 - 21 Loss \n",
      "Win 4 - 21 Loss \n",
      "Win 5 - 21 Loss \n",
      "Win 6 - 21 Loss \n",
      "190\n",
      "Win 1 - 21 Loss \n",
      "Win 3 - 21 Loss \n",
      "Win 6 - 21 Loss \n",
      "Win 5 - 21 Loss \n",
      "Win 5 - 21 Loss \n",
      "195\n",
      "Win 6 - 21 Loss \n",
      "Win 6 - 21 Loss \n",
      "Win 5 - 21 Loss \n",
      "Win 6 - 21 Loss \n",
      "Win 2 - 21 Loss \n"
     ]
    }
   ],
   "source": [
    "# train model:\n",
    "# looking for value of reward 1 and -1:\n",
    "MAX_N_GAMES = 200\n",
    "n_iteration = 200\n",
    "Q.update_epsilon(0.1)\n",
    "Q.update_optimizer(lr=1e-5)\n",
    "Q.min_loss = 0.1\n",
    "Q.max_iter = 4\n",
    "for episode in range(MAX_N_GAMES):\n",
    "    if episode % 5 == 0:\n",
    "        print(episode)\n",
    "    observation = env.reset()\n",
    "    \n",
    "    generate_game(1, D, Q, preprocessor, env, is_e_greedy=True, verbose=False, is_update_epsilon=False)\n",
    "\n",
    "    train_model(n_iteration, verbose=False)\n",
    "        \n",
    "    Q_hat.update_state_dict(Q)\n",
    "    Q.update_epsilon()\n",
    "    Q.update_min_loss()\n",
    "Q.save_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "_ = env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "_ = play(env, Q, preprocessor, False, render=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Win 6 - 21 Loss \n",
      "Win 6 - 21 Loss \n",
      "Win 4 - 21 Loss \n",
      "Win 15 - 21 Loss \n",
      "Win 4 - 21 Loss \n"
     ]
    }
   ],
   "source": [
    "generate_game(5, D, Q, preprocessor, env, is_e_greedy=False, verbose=False, is_update_epsilon=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}