MultiArmed Bandit.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Reinforcement Learning :\n",
    "\n",
    "MultiArmed Bandit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "import logging"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating environment multiarmed bandit\n",
    "class MultiArmedBandit:\n",
    "    # 4 armed bandit\n",
    "    def __init__(self):\n",
    "        # Probabilities of payout for 4 arms.\n",
    "        self.bandit = [0.2, 0.0, 0.1, -4.0]\n",
    "        self.num_actions = 4\n",
    "    \n",
    "    # Pull an arm if random number > bandit[arm] number i.e if random number > prob of arm reward is 1 else -1.\n",
    "    def pull(self, arm):\n",
    "        return 1 if np.random.randn(1) > self.bandit[arm] else -1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Agent class which will work with MultiArmedBandit.\n",
    "class Agent:\n",
    "    def __init__(self, actions = 4):\n",
    "        self.num_actions = actions\n",
    "        self.reward_in = tf.placeholder(tf.float32, [1], name='reward_in')\n",
    "        self.action_in = tf.placeholder(tf.int32, [1], name='action_in')\n",
    "        \n",
    "        self.W = tf.get_variable('W', [self.num_actions])\n",
    "        self.best_action = tf.argmax(self.W, axis=0)\n",
    "        \n",
    "        actions_weight = tf.slice(self.W, self.action_in, [1])\n",
    "        policy_loss = -(tf.log(actions_weight) * self.reward_in)\n",
    "        self.optimizer = tf.train.AdamOptimizer(learning_rate=1e-3).minimize(policy_loss)\n",
    "        \n",
    "    def predict(self, sess):\n",
    "        return sess.run(self.best_action)\n",
    "    \n",
    "    def random_or_predict(self, sess, epsilon):\n",
    "        if np.random.rand(1) < epsilon:\n",
    "            return np.random.randint(self.num_actions)\n",
    "        else:\n",
    "            return self.predict(sess)\n",
    "        \n",
    "    def train(self, sess, action, reward):\n",
    "        sess.run(self.optimizer, {self.action_in:[action], self.reward_in:[reward]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From c:\\program files\\python36\\lib\\site-packages\\tensorflow\\python\\framework\\op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n"
     ]
    }
   ],
   "source": [
    "env = MultiArmedBandit()\n",
    "agent = Agent()\n",
    "num_episodes = 50000\n",
    "EPSILON = 0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3\n",
      "1\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    for _ in range(num_episodes):\n",
    "        action = agent.random_or_predict(sess, EPSILON)\n",
    "        reward = env.pull(action)\n",
    "        agent.train(sess, action, reward)\n",
    "        \n",
    "    print(np.argmin(np.array(env.bandit)))\n",
    "    print(agent.predict(sess))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}