3-Q-Network-TD.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Exercise 3: Q-Learning with TD(0) Updates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from __future__ import division\n",
    "\n",
    "import numpy as np\n",
    "import random\n",
    "import os\n",
    "import tensorflow as tf\n",
    "import tensorflow.contrib.slim as slim\n",
    "\n",
    "from unityenv import UnityEnvironment"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y = .99 # Discount rate\n",
    "start_e = 1 # Initial epsilon value\n",
    "end_e = 0.1 # Final epsilon value\n",
    "annealing_steps = 10000 # Steps to anneal epsilon downward\n",
    "num_episodes = 1000 # Total number of episodes to run environment\n",
    "summary_path = './summaries/q-td' # Path to save summary statistics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load the environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "env_config = {'--grid-size': 7, '--num-objects': 4, '--num-goals': 1}\n",
    "env = UnityEnvironment(file_name=\"FixedGridWorld\", worker_num=2, config=env_config)\n",
    "print (str(env))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Examine the state space"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "_, state = env.reset()\n",
    "print(state)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "State (s) is an integer which corresponds to a discrete state."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## The Q-Learning Agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class QAgent(object):\n",
    "    def __init__(self, num_states, num_actions, lr):\n",
    "        # These lines establish the feed-forward part of the network used to estimate Q(s, a)\n",
    "        self.state_input = tf.placeholder(shape=[1],dtype=tf.int32)\n",
    "        state = slim.one_hot_encoding(self.state_input, num_states)\n",
    "        self.q_out = slim.fully_connected(state, num_actions,\n",
    "                                          biases_initializer=None, \n",
    "                                          weights_initializer= tf.zeros_initializer(),\n",
    "                                          activation_fn=None)\n",
    "        self.predict = tf.argmax(self.q_out,1)\n",
    "\n",
    "        # Below we obtain the loss by taking the sum of squares difference\n",
    "        # between the target and prediction Q values.\n",
    "        self.q_next = tf.placeholder(shape=[1,4],dtype=tf.float32)\n",
    "        self.loss = tf.reduce_sum(tf.squared_difference(self.q_next, self.q_out))\n",
    "        trainer = tf.train.AdamOptimizer(learning_rate=lr)\n",
    "        self.update = trainer.minimize(self.loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training the network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "agent = QAgent(env.state_space_size, env.action_space_size, 1e-2)\n",
    "init = tf.global_variables_initializer()\n",
    "\n",
    "if not os.path.exists(summary_path):\n",
    "    os.makedirs(summary_path)\n",
    "\n",
    "e_drop = (start_e - end_e) / annealing_steps\n",
    "e = start_e    \n",
    "\n",
    "value_table = np.zeros([env.state_space_size])\n",
    "# Create lists to contain total rewards and steps per episode\n",
    "episode_list = []\n",
    "reward_list = []\n",
    "loss_list = []\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    summary_writer = tf.summary.FileWriter(summary_path)\n",
    "    for i in range(num_episodes):\n",
    "        # Reset environment and get first new observation\n",
    "        _, state = env.reset()\n",
    "        total_reward = 0\n",
    "        done = False\n",
    "        steps = 0\n",
    "        while not done:\n",
    "            steps +=1\n",
    "            # Choose an action by greedily (with e chance of random action) from the Q-network\n",
    "            action, Q = sess.run([agent.predict,agent.q_out],feed_dict={agent.state_input: state})\n",
    "            action = action[0]\n",
    "            if np.random.rand(1) < e:\n",
    "                action = np.random.randint(0, env.action_space_size)\n",
    "            \n",
    "            # Get new state and reward from environment\n",
    "            _, state_1, reward, done = env.step(action, value_table.tolist())\n",
    "            \n",
    "            targetQ = Q\n",
    "            if not done:\n",
    "                #Obtain the Q' values by feeding the new state through our network\n",
    "                Q1 = sess.run(agent.q_out,feed_dict={agent.state_input: state_1})\n",
    "                #Obtain maxQ' and set our target value for chosen action.\n",
    "                maxQ1 = np.max(Q1)\n",
    "                targetQ[0, action] = reward + y*maxQ1\n",
    "            else:\n",
    "                targetQ = Q\n",
    "                targetQ[0, action] = reward\n",
    "            \n",
    "            # Train our network using target and estimated Q values\n",
    "            _, q_table, v_loss = sess.run([agent.update,tf.trainable_variables()[0], agent.loss],\n",
    "                             feed_dict={agent.state_input: state, \n",
    "                                        agent.q_next: targetQ})\n",
    "            total_reward += reward\n",
    "            state = state_1\n",
    "            value_table = np.mean(q_table, axis=1)\n",
    "            if e > end_e:\n",
    "                e -= e_drop\n",
    "            loss_list.append(v_loss)\n",
    "        episode_list.append(steps)\n",
    "        reward_list.append(total_reward)\n",
    "        if i % 50 == 0 and i != 0:\n",
    "            summary = tf.Summary()\n",
    "            summary.value.add(tag='Info/Reward', simple_value=float(np.mean(reward_list[-50:])))\n",
    "            summary.value.add(tag='Info/Value Loss', simple_value=float(np.mean(loss_list[-50:])))\n",
    "            summary.value.add(tag='Info/Epsilon', simple_value=float(e))\n",
    "            summary.value.add(tag='Info/Q Estimate', simple_value=float(np.mean(value_table)))\n",
    "            summary.value.add(tag='Info/Episode Length', simple_value=float(np.mean(episode_list[-50:])))\n",
    "            summary_writer.add_summary(summary, i)\n",
    "            summary_writer.flush()\n",
    "\n",
    "            print (\"Mean Reward: {}\".format(np.mean(reward_list[-50:])))\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "env.close()"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}