Policy_Grad_Run.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import keras.backend as K\n",
    "from keras import Sequential, Model\n",
    "from keras.layers import Input, Dense, Conv2D, MaxPooling2D, Dropout, Flatten\n",
    "import copy\n",
    "\n",
    "import Environment as E\n",
    "import importlib\n",
    "import matplotlib.pyplot as plt\n",
    "import Heatmap as H\n",
    "import GaussianTrashSource"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# General\n",
    "\n",
    "n_agents = 1\n",
    "\n",
    "# Model\n",
    "\n",
    "time_steps = 3\n",
    "grid_size_w = 7\n",
    "grid_size_h = 7\n",
    "n_actions = 5\n",
    "\n",
    "n_channels = time_steps + time_steps + 1\n",
    "number_print = 25\n",
    "\n",
    "#Policy Gradient\n",
    "n_episodes = 250\n",
    "n_steps = 30\n",
    "gamma = 0.9\n",
    "\n",
    "env_reset_freq = 0 #Environment/trash locations change every env_reset_freq episodes\n",
    "n_runs = 10\n",
    "heatmap = H.Heatmap(keep_track_of_steps=1000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Policy Gradient"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from Motion import Motion\n",
    "\n",
    "def tuple_to_batch(known_data):\n",
    "    batch = []\n",
    "    for i,agent_pos in enumerate(known_data[2]):\n",
    "        batch.append(np.concatenate((known_data[0],known_data[1],known_data[2][i:i+1])))\n",
    "    batch = np.array(batch)\n",
    "    return batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_i = Input(shape=(n_channels, grid_size_w, grid_size_h))\n",
    "conv_1 = Conv2D(16, (2, 2), activation=\"relu\", data_format=\"channels_first\")(data_i)\n",
    "conv_2 = Conv2D(32, (2, 2), activation=\"relu\", data_format=\"channels_first\")(conv_1)\n",
    "pool_1 = MaxPooling2D(data_format=\"channels_first\")(conv_2)\n",
    "drop_1 = Dropout(0.25)(pool_1)\n",
    "flat_1 = Flatten()(drop_1)\n",
    "feed_1 = Dense(64, activation=\"relu\")(flat_1)\n",
    "drop_2 = Dropout(0.25)(feed_1)\n",
    "feed_2 = Dense(n_actions, activation=\"linear\")(drop_2)\n",
    "data_o = feed_2\n",
    "\n",
    "model = Model(inputs=data_i, outputs=data_o)\n",
    "\n",
    "### THE ONLY DIFFERENCE FROM Q-Learning loss=policy_loss\n",
    "def policy_loss(yPred,reward_sums):\n",
    "    return  - K.log(yPred + 1)*reward_sums\n",
    "\n",
    "model.compile(optimizer=\"Adadelta\", loss=policy_loss)\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "np.random.seed(0)\n",
    "list_avg_rwd = []\n",
    "for i in range(n_episodes):\n",
    "    if i==env_reset_freq:\n",
    "        env = E.Environment(dim = (grid_size_h,grid_size_w),reward_eat_trash=10, \n",
    "                            reward_invalid_move=0, reward_nothing_happend=0, \n",
    "                            trash_appearence_prob=1, number_trash_sources=3, saved_timesteps=time_steps )\n",
    "        _, _,  _, _, trash_sources = env.debug_data_export()\n",
    "        for idx,trash_source in enumerate(trash_sources): \n",
    "            print(\"Trash Source {} has its mean at {}\".format(idx, trash_source.mean))\n",
    "        d = []\n",
    "        action_avg = np.zeros((10000,5))\n",
    "        list_steps = []\n",
    "        all_time_reward = 0\n",
    "        all_time_reward_avg_history = []\n",
    "        all_time_reward_steps_history = []\n",
    "        for k in range(n_agents):\n",
    "            env.add_agent(coord=(k,0),capacity=100000)\n",
    "        r_sum = 0\n",
    "    \n",
    "    if i % 10 == 0:\n",
    "        print(\"Episode {} of {}\".format(i + 1, n_episodes))\n",
    "    \n",
    "    X = tuple_to_batch(env.export_known_data()) \n",
    "    \n",
    "    R = np.zeros((n_runs, n_steps, n_agents, n_actions))\n",
    "    X_runs = np.zeros((n_runs, n_steps, n_agents, grid_size_w, grid_size_h, n_channels))\n",
    "\n",
    "    for run in range(n_runs):\n",
    "        action_history = np.zeros((n_steps, n_agents))\n",
    "        for step in range(n_steps):\n",
    "            #Determine policy decisions in current state\n",
    "            P_vals = model.predict(X)\n",
    "            actions = np.random.choice(5,1,P_vals.tolist())\n",
    "\n",
    "            #New known state and rewards\n",
    "            X_reward = env.move_agents(actions)\n",
    "            heatmap.add_map(X_reward[1][-1]) #Get the current position of all agents\n",
    "            \n",
    "            #After all agents move\n",
    "            X_new = tuple_to_batch(X_reward[:3])\n",
    "            rewards = X_reward[3]\n",
    "\n",
    "            all_time_reward += sum(rewards)\n",
    "            action_avg[step%number_print, : ] = 0\n",
    "            action_avg[step%number_print,actions[0]]= 1\n",
    "            d.append(sum(rewards))\n",
    "            while(len(d)>number_print):\n",
    "                del(d[0])\n",
    "            if(step% number_print == 0 and i%10==0): \n",
    "                mean = sum(d) / number_print\n",
    "                \n",
    "                print('Current episode is', i)\n",
    "                print(\"In Step {} the average reward of {} is {} \".format(step, number_print, mean))\n",
    "                print(\"Actions: {}\".format(np.sum(action_avg, axis = 0)))\n",
    "                list_avg_rwd.append(mean)\n",
    "                list_steps.append(step)\n",
    "                print(heatmap.get_heatmap()) #Heatmap\n",
    "\n",
    "\n",
    "            for agent_i in range(n_agents):\n",
    "                action_history[step,agent_i] = int(actions[agent_i]) #copy actions to history\n",
    "                X_runs[run,step,agent_i] = copy.copy(X[agent_i]) #copy current state to X_runs\n",
    "                R[run, step, agent_i, actions[agent_i]] = rewards[agent_i] #copy current reward to runs reward\n",
    "                #update previous rewards in R matrix:\n",
    "                for j,prev in enumerate(range(max([0,step-15]),step).__reversed__()):\n",
    "                    R[run, prev, agent_i, int(action_history[prev,agent_i])] += rewards[agent_i] * (gamma ** j)\n",
    "                    \n",
    "#This block uses all rewards for each action instead of discounting. This may be used instead of previous 2 lines\n",
    "#                 for j,prev in enumerate(range(step).__reversed__()):\n",
    "#                     R[run, prev, agent_i, int(action_history[prev,agent_i])] += rewards[agent_i]\n",
    "                    \n",
    "            #update state\n",
    "            X = X_new\n",
    "\n",
    "    #epochs may be changed\n",
    "    X_fit = X_runs.reshape((n_runs * n_steps * n_agents, grid_size_w, grid_size_h, n_channels))\n",
    "    R_fit = R.reshape((n_runs * n_steps * n_agents, n_actions))\n",
    "    model.fit(X_fit, R_fit, epochs=1, verbose=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(list_avg_rwd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sum(list_avg_rwd[:50])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}