Experiment1-SH.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "from engine import RMG\n",
    "from agent import RandomAgent, IndQLearningAgent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n"
     ]
    }
   ],
   "source": [
    "N_EXP = 10\n",
    "\n",
    "r0ss = []\n",
    "r1ss = []\n",
    "\n",
    "for n in range(N_EXP):\n",
    "\n",
    "    batch_size = 1\n",
    "    max_steps = 20\n",
    "    gamma = 0.96\n",
    "\n",
    "    # Reward matrix for the Iterated Stag Hunt\n",
    "    ipd_rewards = np.array([[2., 1.], [0., 1.]])\n",
    "\n",
    "\n",
    "    env = RMG(max_steps=max_steps, payouts=ipd_rewards, batch_size=batch_size)\n",
    "    env.reset()\n",
    "\n",
    "    possible_actions = [0, 1]  # Cooperate or Defect\n",
    "\n",
    "    #cooperator, defector = RandomAgent(possible_actions, p=0.9), RandomAgent(possible_actions, p=0.1)\n",
    "    cooperator, defector = IndQLearningAgent(possible_actions, n_states=1, learning_rate=0.3, epsilon=0.1, gamma=gamma), \\\n",
    "        IndQLearningAgent(possible_actions, n_states=1, learning_rate=0.3, epsilon=0.1, gamma=gamma)\n",
    "\n",
    "\n",
    "    # Stateless interactions (agents do not have memory)\n",
    "    s = 0\n",
    "\n",
    "\n",
    "    n_iter = 1000\n",
    "\n",
    "    r0s = []\n",
    "    r1s = []\n",
    "\n",
    "    for i in range(n_iter):\n",
    "\n",
    "        # A full episode:\n",
    "        done = False\n",
    "\n",
    "        while not done:\n",
    "\n",
    "            # Agents decide\n",
    "            a0 = cooperator.act()\n",
    "            a1 = defector.act()\n",
    "\n",
    "            # World changes\n",
    "            _, (r0, r1), done, _ = env.step(([a0], [a1]))\n",
    "\n",
    "            # Agents learn\n",
    "\n",
    "            cooperator.update(s, (a0, a1), (r0, r1), s )\n",
    "            defector.update(s, (a1, a0), (r1, r0), s )\n",
    "\n",
    "            #s = new_s  #stateless!\n",
    "\n",
    "            #print(r0, r1)\n",
    "            r0s.append(r0[0])\n",
    "            r1s.append(r1[0])\n",
    "            \n",
    "            cooperator.epsilon *= 0.99\n",
    "            defector.epsilon *= 0.99\n",
    "\n",
    "        env.reset()\n",
    "        \n",
    "    print(n)\n",
    "    r0ss.append(r0s)\n",
    "    r1ss.append(r1s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def moving_average(a, n=3) :\n",
    "    ret = np.cumsum(a, dtype=float)\n",
    "    ret[n:] = ret[n:] - ret[:-n]\n",
    "    return ret[n - 1:] / n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We report moving avearage of rewards, since it's common in RL taks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZ0AAAENCAYAAADUubjhAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAHotJREFUeJzt3XtwVPXh/vFnd5MQQkLMRQLhUg2XCloEDGDDl4uyZBCpMBRB2lLRWvUXhKKVipeiVnHiJcKgYbQjojL9A1olTGuLutJIBbmEEFBQIICjDpeYbAgJJCHJnt8fDDuEJFzPfs4mvF8znbJ7Pjn7nLPrPjmXnOOyLMsSAAAGuJ0OAAC4clA6AABjKB0AgDGUDgDAGEoHAGAMpQMAMCbC6QCSVFpaqtzcXB09elQul0ter1fjxo1rNGbnzp166aWX1KlTJ0nS0KFDNXnyZCfiAgAuUViUjsfj0fTp05WWlqbq6mrNmzdP/fv3V7du3RqN69u3r+bNm+dQSgDA5QqL3WsJCQlKS0uTJLVv315du3aV3+93OBUAwG5hsaVzppKSEh04cEC9evVqMm3Pnj2aO3euEhISNH36dHXv3r3JGJ/PJ5/PJ0nKzs4OeV4AwIVzhdNlcGpqavT0009r0qRJGjp0aKNpJ06ckNvtVnR0tAoLC/XOO+9o8eLF553nwYMHQxXXNsnJySotLXU6xnmR017ktFdryNkaMkpSampqyOYdFrvXJKm+vl45OTkaPnx4k8KRpJiYGEVHR0uSBg0apIaGBh07dsx0TADAZQiL0rEsS2+88Ya6du2q8ePHNzvm6NGjOr1RVlxcrEAgoLi4OJMxAQCXKSyO6ezevVvr1q1Tjx49NHfuXEnStGnTgpuhmZmZ2rhxoz7++GN5PB5FRUVpzpw5crlcTsYGAFyksCid6667TitXrjznmLFjx2rs2LGGEgEAQiEsdq8BAK4MlA4AwBhKBwBgDKUDADCG0gEAGEPpAACMoXQAAMZQOgAAYygdAIAxlA4AwBhKBwBgDKUDADCG0gEAGEPpAACMoXQAAMZQOgAAYygdAIAxlA4AwBhKBwBgDKUDADCG0gEAGEPpAACMoXQAAMZQOgAAYygdAIAxlA4AwBhKBwBgTITTASSptLRUubm5Onr0qFwul7xer8aNG9dojGVZWrZsmbZt26Z27dopKytLaWlpDiUGAFyKsCgdj8ej6dOnKy0tTdXV1Zo3b5769++vbt26Bcds27ZNhw8f1uLFi7V371699dZbeuGFFxxMDQC4WGFROgkJCUpISJAktW/fXl27dpXf729UOgUFBRoxYoRcLpf69Omj48ePq7y8PPhzLamvbVCgrkGSVHusVnK5VFlao9ryWlkxHeSpOKr4+ICqq12qrQ3dMp5LRUK5ysuPOvPiF4Gc9iKnvVpDztaQ8apeVyk1NTVk8w+L0jlTSUmJDhw4oF69ejV63u/3Kzk5Ofg4KSlJfr//vKVT/GGxyr7xK7Buq1R+TBX1cYo9eVQuq0FyuSTLOv1/jvne7VYgEHAuwAUip73Iaa/WkLM1ZDw8/Rb1y7g+ZPMPq9KpqalRTk6OZsyYoZiYmEuah8/nk8/nkyRlZ2fLtW2f9M8CuSyX5JKSIyukazqrQ89OcrWLUFVdpGojO8rtstQhVnI5cGqFx+NWQ0N4fxAlctqNnPZqDTlbQ8be424I6fzDpnTq6+uVk5Oj4cOHa+jQoU2mJyYmqrS0NPi4rKxMiYmJTcZ5vV55vd7g49IPtyhgSSUp1+m6qWnq1DNWcd3jFZ3cQYqMVG29W4GIdpKk6PanNn5MS05ObrRs4Yqc9iKnvVpDztaQMdTConQsy9Ibb7yhrl27avz48c2OSU9P15o1azRs2DDt3btXMTEx5921dmreLh2NS1X6n3+uq2/opJiEdo2mt2vh5wAA9guL0tm9e7fWrVunHj16aO7cuZKkadOmBX8jyMzM1MCBA1VYWKjZs2crKipKWVlZFzTvQMBSt/83Qol9UxSTEBWyZQAAnF9YlM51112nlStXnnOMy+XSfffdd9HzropJVo/eSfJEh8WiAsAVrc1fkeBE12sUHedRVDsHDtYAABpp86XTaWhXJV4Tp4hISgcAnNbmS6f74CRFRnucjgEA0BVQOh2vTXTkNGgAQFNtvnQSOnPGGgCEizZfOu4ozloDgHDR5kuHfWsAED7afukAAMIGpQMAMIbSAQAYQ+kAAIyhdAAAxlA6AABjKB0AgDGUDgDAGEoHAGAMpQMAMIbSAQAYQ+kAAIyhdAAAxlA6AABjKB0AgDGUDgDAGEoHAGAMpQMAMIbSAQAYQ+kAAIyhdAAAxkQ4HeC0JUuWqLCwUPHx8crJyWkyfefOnXrppZfUqVMnSdLQoUM1efJk0zEBAJchbEpn1KhRGjt2rHJzc1sc07dvX82bN89gKgCAncJm91q/fv0UGxvrdAwAQAiFzZbOhdizZ4/mzp2rhIQETZ8+Xd27d28yxufzyefzSZKys7OVnJxsOuZFi4iIIKeNyGkvctqnNWQMNZdlWZbTIU4rKSnRiy++2OwxnRMnTsjtdis6OlqFhYV65513tHjx4vPO8+DBg6GIaqvk5GSVlpY6HeO8yGkvctqrNeRsDRklKTU1NWTzDpvda+cTExOj6OhoSdKgQYPU0NCgY8eOOZwKAHAxWk3pHD16VKc3yoqLixUIBBQXF+dwKgDAxQibYzqLFi3Srl27VFlZqQcffFBTpkxRfX29JCkzM1MbN27Uxx9/LI/Ho6ioKM2ZM0cul8vh1ACAixE2pTNnzpxzTh87dqzGjh1rKA0AIBRaze41AEDrR+kAAIyhdAAAxlA6AABjKB0AgDGUDgDAGEoHAGAMpQMAMIbSAQAYQ+kAAIyhdAAAxlA6AABjKB0AgDGUDgDAGEoHAGAMpQMAMIbSAQAYQ+kAAIyhdAAAxlA6AABjKB0AgDGUDgDAGEoHAGAMpQMAMIbSAQAYQ+kAAIyhdAAAxkQ4HeC0JUuWqLCwUPHx8crJyWky3bIsLVu2TNu2bVO7du2UlZWltLQ0B5ICAC5V2GzpjBo1Sk888USL07dt26bDhw9r8eLFuv/++/XWW28ZTAcAsMNllc53332nV1991ZYg/fr1U2xsbIvTCwoKNGLECLlcLvXp00fHjx9XeXm5La8NADDjvLvXamtrtWrVKn377bfq0qWL7rzzTlVWVuq9997Tjh07NHLkSBM55ff7lZycHHyclJQkv9+vhISERuN8Pp98Pp8kKTs7u9HPhKuIiAhy2oic9iKnfVpDxlA7b+ksXbpUBw4c0I033qiioiJ99913OnjwoEaOHKkHHnhAHTt2NJHzgnm9Xnm93uDj0tJSB9NcmOTkZHLaiJz2Iqd9WkNGSUpNTQ3ZvM9bOtu3b9dLL72k+Ph43XbbbcrKytIzzzyjvn37hixUcxITExu9WWVlZUpMTDSaAQBwec57TKempkbx8fGSTu3Sio6ONl44kpSenq5169bJsizt2bNHMTExTXatAQDC23m3dBoaGvTVV181eu7sxzfccMNlB1m0aJF27dqlyspKPfjgg5oyZYrq6+slSZmZmRo4cKAKCws1e/ZsRUVFKSsr67JfEwBglsuyLOtcA2bOnHnuGbhcev31120NZaeDBw86HeG8Wst+XnLai5z2ag05W0NGyeFjOrm5uSF7cQDAlSVs/jgUAND2UToAAGMoHQCAMZQOAMAYSgcAYAylAwAwhtIBABhD6QAAjKF0AADGUDoAAGMoHQCAMZQOAMAYSgcAYAylAwAwhtIBABhD6QAAjKF0AADGUDoAAGMoHQCAMZQOAMAYSgcAYAylAwAwhtIBABhD6QAAjKF0AADGUDoAAGMinA5wWlFRkZYtW6ZAIKDRo0dr4sSJjabn5+dr+fLlSkxMlCSNHTtWo0ePdiIqAOAShUXpBAIBLV26VE899ZSSkpL0+OOPKz09Xd26dWs0LiMjQ7/73e8cSgkAuFxhsXutuLhYnTt3VkpKiiIiIpSRkaEtW7Y4HQsAYLOw2NLx+/1KSkoKPk5KStLevXubjNu0aZO+/vprdenSRXfffbeSk5ObjPH5fPL5fJKk7OzsZseEm4iICHLaiJz2Iqd9WkPGUAuL0rkQN910k4YNG6bIyEh98sknys3N1dNPP91knNfrldfrDT4uLS01GfOSJCcnk9NG5LQXOe3TGjJKUmpqasjmHRa71xITE1VWVhZ8XFZWFjxh4LS4uDhFRkZKkkaPHq39+/cbzQgAuHxhUTo9e/bUoUOHVFJSovr6em3YsEHp6emNxpSXlwf/XVBQ0OQkAwBA+AuL3Wsej0f33nuvFixYoEAgoFtuuUXdu3fXihUr1LNnT6Wnp+s///mPCgoK5PF4FBsbq6ysLKdjAwAuksuyLMvpEKF08OBBpyOcV2vZz0tOe5HTXq0hZ2vIKF0Bx3QAAFcGSgcAYAylAwAwhtIBABhD6QAAjKF0AADGUDoAAGMoHQCAMZQOAMAYSgcAYAylAwAwhtIBABhD6QAAjKF0AADGUDoAAGMoHQCAMZQOAMAYSgcAYAylAwAwhtIBABhD6QAAjKF0AADGUDoAAGMoHQCAMZQOAMAYSgcAYAylAwAwJsLpAKcVFRVp2bJlCgQCGj16tCZOnNhoel1dnV5//XXt379fcXFxmjNnjjp16uRQWgDApQiLLZ1AIKClS5fqiSee0MKFC7V+/Xr98MMPjcasXbtWHTp00Guvvabbb79df/vb3xxKCwC4VGGxpVNcXKzOnTsrJSVFkpSRkaEtW7aoW7duwTEFBQW68847JUk333yz3n77bVmWJZfLdVGvZVmWampqFAgELvpnQ+XIkSOqra11OkYjlmXJ7XYrOjo6bNYTgNYvLErH7/crKSkp+DgpKUl79+5tcYzH41FMTIwqKyvVsWPHi3qtmpoaRUZGKiIiLBZdkhQRESGPx+N0jCbq6+tVU1Oj9u3bOx0FQBsRPt+8NvH5fPL5fJKk7OxsJScnN5p+5MgRtWvXzolo5xROJXhaRESEXC5XcB1GREQ0WZ/hiJz2Iqd9WkPGUAuLb7rExESVlZUFH5eVlSkxMbHZMUlJSWpoaNCJEycUFxfXZF5er1derzf4uLS0tNH02trasNuqiIiIUH19vdMxmlVbWxtch8nJyU3WZzgip73IaZ/WkFGSUlNTQzbvsDiRoGfPnjp06JBKSkpUX1+vDRs2KD09vdGYm266Sfn5+ZKkjRs36vrrr+dYAwC0MmFROh6PR/fee68WLFighx9+WD//+c/VvXt3rVixQgUFBZKkW2+9VVVVVZo1a5b+9a9/6de//rXDqS/PmjVr1LVrVxUXF4dk/l999ZU+/fTTc46ZP3++brrpJgUCgZBkAICzhcXuNUkaNGiQBg0a1Oi5qVOnBv8dFRWlRx55xHSskMnLy9OQIUOUl5enefPm2T7/nTt3aseOHRo9enSz0wOBgNasWaMuXbroiy++0LBhw2zPAABnC5vSccL8+R21a1ekrfPs169Of/nLsXOOOX78uLZs2aKVK1dqxowZwdIJBAJ68skntX79eqWmpioyMlJTp07V+PHjtWPHDj377LM6fvy4EhMTtXDhQqWkpGjy5MkaOHCgNmzYoIqKCuXk5GjgwIF65ZVXVFNTo82bN+uhhx7ShAkTGmXYsGGDfvrTn+oXv/iFVq9eTekAMCIsdq9daT766CONGjVKPXv2VEJCgrZv3y5J+ve//60ffvhB+fn5Wrx4sbZu3Srp1NUYnnrqKf31r3/VmjVrNHXqVL344ovB+dXX1+vDDz/Us88+q1dffVVRUVF69NFHdccdd+iTTz5pUjiStHr1ak2YMEG33XabPv30U9XV1ZlZeABXtCt6S+d8WyShkpeXp/vuu0+SNGHCBK1atUrXX3+9Nm/erPHjx8vtdqtTp07KyMiQJO3bt0+7d+/WXXfdJenUFtGZlwAaN26cJKl///5NruTQnJMnT2rt2rV6+umnFRsbq4EDByo/P19jxoyxe1EBoJErunScUF5ervXr1+ubb76Ry+VSQ0OD3G63nnzyyRZ/xrIs9enTR//85z+bnR4VFSXp1AkZF3LqdX5+vioqKoLHe6qrqxUdHU3pAAg5dq8Z9uGHH+qXv/ylNm/erE2bNqmgoEA9evTQpk2bNHjwYH344YcKBAL68ccf9cUXX0g6dUq53+8PnslXV1en3bt3n/N1YmNjVVVV1ey01atX65VXXtGmTZu0adMmbdy4UevWrVN1dbW9CwsAZ6F0DMvLy9Ntt93W6Lnbb79deXl5uv3229WlSxeNGjVKs2bN0g033KCOHTsqKipKb775pl544QV5vV5lZmYGC6glGRkZ2rt3r8aMGaPVq1cHn6+urlZ+fn6js9piYmI0ZMgQffzxx/YuLACcxWVZluV0iFA6ePBgo8cnTpxQTEyMQ2mad+YVCY4fP64OHTrI7/dr/PjxysvLc/QWDmeur9by19TktBc57dMaMkqhvSIBx3TCzN13362KigrV1dXpD3/4A/cMAtCmUDph5h//+IfTEQAgZDimAwAwhtIBABhD6QAAjKF0AADGUDoOcfLWBhs2bNB1112nMWPGyOv1aurUqa3iNE4ArR+l45Azb20QCjt37tTatWtbnD5kyBB98skn8vl8GjBggN55552Q5ACAM13Rp0x3nD9fkbt22TrPun79dOwvfznnmHC4tcFplmWpqqpK11xzja3rAQCaw5aOA8Lh1gabN2/WmDFjNHjwYP3vf/8LXsEaAELpit7SOd8WSag4fWsD6dTutffee0+SlJubq+eff75RkQFAKFzRpeOEcLi1wdkyMzP1+9///qJ/DgAuFrvXDAuHWxucbfPmzfrJT35yeQsGABeA0jHM6VsbnHb6mI7X69X777+v+fPn27qcANAcbm0QBri1gb3IaS9y2qc1ZJS4tcEVhVsbAGjLKJ0ww60NALRlV9wxnTa+N9F2rC8AdrriSsftdl/SacVXovr6erndV9xHBEAIXXG716Kjo1VTU6Pa2lq5XC6n40iS2rVrp9raWqdjNGJZltxut6Kjo52OAqANueJKx+VyqX379k7HaKS1nNECAJfL8dKpqqrSwoUL9eOPP+rqq6/Www8/rNjY2Cbjpk6dqh49ekg69SX92GOPmY4KALhMjpdOXl6efvazn2nixInKy8tTXl6efvOb3zQZFxUVpZdfftmBhAAAuzh+lHjLli0aOXKkJGnkyJHasmWLw4kAAKHi+JZORUWFEhISJElXXXWVKioqmh1XV1enefPmyePxaMKECRoyZEiz43w+n3w+nyQpOzs7pH9Zaydy2ouc9iKnfVpDxlAysqXz3HPP6Y9//GOT/529VeNyuVo8o2zJkiXKzs7W7Nmz9e677+rw4cPNjvN6vcrOzlZ2dnbw5mjhjpz2Iqe9yGmf1pBRCm1OI1s6f/7zn1ucFh8fr/LyciUkJKi8vFwdO3ZsdlxiYqIkKSUlRf369dO3336rzp07hyQvACA0HD+mk56ers8++0yS9Nlnn2nw4MFNxlRVVamurk6SdOzYMe3evVvdunUzmhMAcPk8zzzzzDNOBkhLS9Pq1av1/vvvq6qqSvfcc4+ioqK0b98+rVy5Uunp6Tpw4IBefPFF+Xw+5efn64477tCNN954wfNvDchpL3Lai5z2aQ0ZpdDlbPO3NgAAhA/Hd68BAK4clA4AwBjH/04nVIqKirRs2TIFAgGNHj1aEydONPbapaWlys3N1dGjR+VyueT1ejVu3DitXLlSn376afAMvWnTpmnQoEGSpFWrVmnt2rVyu9265557NGDAACPLMXPmTEVHR8vtdsvj8Sg7O7vFSxNZlqVly5Zp27ZtateunbKysoL7ffPz8/XBBx9IkiZNmqRRo0bZlvHgwYNauHBh8HFJSYmmTJmi48ePO74+lyxZosLCQsXHxysnJ0dSy5d2upT1t3//fuXm5urkyZMaOHCg7rnnnku6UG1zOZcvX66tW7cqIiJCKSkpysrKUocOHVRSUqKHH344+PckvXv31v3333/OPBd6OatLyWnnfzclJSVatGiRKisrlZaWplmzZiki4uK/BpvLuXDhwuCdik/fcffll192bH229D3k+OfTaoMaGhqshx56yDp8+LBVV1dnPfroo9b3339v7PX9fr+1b98+y7Is68SJE9bs2bOt77//3lqxYoW1evXqJuO///5769FHH7VOnjxpHTlyxHrooYeshoYGI8uRlZVlVVRUNHpu+fLl1qpVqyzLsqxVq1ZZy5cvtyzLsrZu3WotWLDACgQC1u7du63HH3/csizLqqystGbOnGlVVlY2+ncoNDQ0WPfdd59VUlISFutz586d1r59+6xHHnkk+Jyd62/evHnW7t27rUAgYC1YsMAqLCy0LWdRUZFVX18fzHw655EjRxqNO1NLeVpaZjty2vk+5+TkWJ9//rllWZb15ptvWh999JFtOc/07rvvWn//+98ty3Jufbb0PeT057NN7l4rLi5W586dlZKSooiICGVkZBi9vE5CQkLwN4T27dura9eu8vv9LY7fsmWLMjIyFBkZqU6dOqlz584qLi52bDlaujRRQUGBRowYIZfLpT59+uj48eMqLy9XUVGR+vfvr9jYWMXGxqp///4qKioKSbYvv/xSnTt31tVXX33O/KbWZ79+/Zr8BmrX+isvL1d1dbX69Okjl8ulESNGXHLe5nLeeOON8ng8kqQ+ffqc8zMq6Zx57LqcVXM5W3Kx77NlWdq5c6duvvlmSdKoUaNCktOyLH3xxRcaNmzYOecR6vXZ0veQ05/PNrl7ze/3KykpKfg4KSlJe/fudSRLSUmJDhw4oF69eumbb77RRx99pHXr1iktLU2//e1vFRsbK7/fr969ewd/JjExMfgFYGI5FixYIEkaM2aMvF5vi5cm8vv9Sk5ObpTH7/c3Wd9n5rfb+vXrG/3HHI7r067119znOFTrde3atcrIyAg+Likp0Z/+9Ce1b99ed911l/r27XvOPBd6OatLZcf7XFlZqZiYmGDRhupz+vXXXys+Pl5dunQJPuf0+jzze8jpz2ebLJ1wUVNTo5ycHM2YMUMxMTHKzMzU5MmTJUkrVqzQe++9p6ysLEczPvfcc0pMTFRFRYWef/75JteFOteliUyrr6/X1q1b9atf/UqSwnJ9ni2c1l9LPvjgA3k8Hg0fPlzSqd+QlyxZori4OO3fv18vv/xy8LjFhbB7mVvD+3yms38xcnp9nv09ZOe8L0Wb3L2WmJiosrKy4OOysrLgZXRMqa+vV05OjoYPH66hQ4dKOvVbhdvtltvt1ujRo7Vv375m8/r9fiUmJhpZjtPzi4+P1+DBg1VcXBy8NJGkRpcmSkxMbHSzudN5Wspvt23btunaa6/VVVddJSk816ck29afibz5+fnaunWrZs+eHfzyiYyMVFxcnKRTfyCYkpKiQ4cOnTNPS8tsB7ve57i4OJ04cUINDQ2NxtupoaFBmzdvbrTV6OT6bO57yOnPZ5ssnZ49e+rQoUMqKSlRfX29NmzYoPT0dGOvb1mW3njjDXXt2lXjx48PPn/6jZakzZs3q3v37pJOXQpow4YNqqurU0lJiQ4dOqRevXqFfDlqampUXV0d/PeOHTvUo0ePFi9NlJ6ernXr1smyLO3Zs0cxMTFKSEjQgAEDtH37dlVVVamqqkrbt28PnkVkp7N/gwy39XmaXesvISFB7du31549e2RZltatW2dr3qKiIq1evVqPPfaY2rVrF3z+2LFjCgQCkqQjR47o0KFDSklJOWeeC7mc1aWy6312uVy6/vrrtXHjRkmnCtfu9//LL79Uampqo91OTq3Plr6HnP58ttkrEhQWFurdd99VIBDQLbfcokmTJhl77W+++Ubz589Xjx49gr89Tps2TevXr9e3334rl8ulq6++Wvfff39w3+oHH3yg//73v3K73ZoxY4YGDhwY8uU4cuSIXnnlFUmnfkP7v//7P02aNEmVlZVauHChSktLm5xSuXTpUm3fvl1RUVHKyspSz549JZ06JrBq1SpJp06pvOWWW2zLKZ0qxaysLL3++uvBXQSvvfaa4+tz0aJF2rVrlyorKxUfH68pU6Zo8ODBtq2/ffv2acmSJTp58qQGDBige++995J2hzSXc9WqVaqvrw8eED99Ku/GjRu1cuVKeTweud1u3XnnncEvk5bytPSZsSPnzp07bXufjxw5okWLFqmqqkrXXnutZs2apcjISFty3nrrrcrNzVXv3r2VmZkZHOvU+mzpe6h3796Ofj7bbOkAAMJPm9y9BgAIT5QOAMAYSgcAYAylAwAwhtIBABhD6QAAjKF0AAfMnDlTO3bscDoGYBylAwAwhj8OBQx77bXX9PnnnysiIkJut1uTJ0/WhAkTnI4FGEHpAA6YOXOmHnjgAfXv39/pKIBR7F4DABhD6QAAjKF0AADGUDqAA6666iqVlJQ4HQMwjhMJAAds2bJFb7/9tqqrqzVp0iTdcccdTkcCjKB0AADGsHsNAGAMpQMAMIbSAQAYQ+kAAIyhdAAAxlA6AABjKB0AgDGUDgDAmP8PEcOaeSj40iEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# We set a fancy theme\n",
    "plt.style.use('ggplot')\n",
    "\n",
    "plt.axis([0, max_steps*n_iter, -0.5, 2.5])\n",
    "for i in range(N_EXP):\n",
    "    plt.plot(moving_average(r0ss[i], 100), 'b', alpha=0.05)\n",
    "    plt.plot(moving_average(r1ss[i], 100), 'r', alpha=0.05)\n",
    "    \n",
    "plt.plot(moving_average(np.asarray(r0ss).mean(axis=0), 100), 'b', alpha=0.5)\n",
    "plt.plot(moving_average(np.asarray(r1ss).mean(axis=0), 100), 'r', alpha=0.5)\n",
    "\n",
    "plt.xlabel('t');\n",
    "plt.ylabel('R');\n",
    "\n",
    "from matplotlib.lines import Line2D\n",
    "cmap = plt.cm.coolwarm\n",
    "custom_lines = [Line2D([0], [0], color='b'),\n",
    "                Line2D([0], [0], color='r')]\n",
    "\n",
    "plt.legend(custom_lines,['Agent A', 'Agent B']);\n",
    "\n",
    "plt.savefig('img/QvsQ_SH.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAasAAAENCAYAAACxVEqeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAHKFJREFUeJzt3X9wFPXhxvHnckcIIfmGXM4kBoglASowIsRYHBQESRlGHWUYxNIprVBEmyhEKhppre1YatBmwhDDWBVRGe1ARfSPtqO9UswoRQMhiiiEABl/JHAmF0hCyEBy+/2D8WpMQn6wd7cL79cMM9nbT/ae3Tvuud3b2zgMwzAEAICFRUU6AAAAvaGsAACWR1kBACyPsgIAWB5lBQCwPMoKAGB5rkgHuFj19fUqLS3VyZMn5XA4lJOTo1tvvbXTmAMHDujpp59WcnKyJGnKlCmaP39+JOICAAbA9mXldDq1aNEiZWRk6MyZMyooKNDEiRM1YsSITuPGjRungoKCCKUEAFwM2x8GTExMVEZGhiRpyJAhGj58uPx+f4RTAQDMZPs9q+/y+Xw6duyYRo8e3WVeVVWVVq1apcTERC1atEgjR47sMsbr9crr9UqSCgsLQ54XANA3jkvlckttbW164oknNG/ePE2ZMqXTvNbWVkVFRSkmJkYVFRV6+eWXtX79+l6XWVtbG6q4pvF4PKqvr490jF6R01zkNI8dMkr2yZmWlhaS5dr+MKAktbe3q6ioSNOmTetSVJIUGxurmJgYSVJWVpY6OjrU1NQU7pgAgAGyfVkZhqHnnntOw4cP1+23397tmJMnT+rbHcjq6moFAgHFx8eHMyYA4CLY/jOrQ4cOqaysTOnp6Vq1apUkaeHChcHd5dmzZ2v37t1699135XQ6FR0drfz8fDkcjkjGBgD0g+3L6uqrr9bWrVsvOGbOnDmaM2dOmBIBAMxm+8OAAIBLH2UFALA8ygoAYHmUFQDA8igrAIDlUVYAAMujrAAAlkdZAQAsj7ICAFgeZQUAsDzKCgBgeZQVAMDyKCsAgOVRVgAAy6OsAACWR1kBACyPsgIAWB5lBQCwPMoKAGB5lBUAwPIoKwCA5VFWAADLo6wAAJZHWQEALI+yAgBYHmUFALA8ygoAYHmuSAe4WPX19SotLdXJkyflcDiUk5OjW2+9tdMYwzC0adMm7du3T4MHD1Zubq4yMjIilBgA0F+2Lyun06lFixYpIyNDZ86cUUFBgSZOnKgRI0YEx+zbt0/Hjx/X+vXrdfjwYb344ov605/+FMHUAID+sH1ZJSYmKjExUZI0ZMgQDR8+XH6/v1NZ7dmzR9OnT5fD4dDYsWN1+vRpNTY2Bn+vN23tbfq65WvFuGLU1t4WkvUYKL/Dr8aTjZGO0Stymouc5rFDRsk+OdPS0kKyXNuX1Xf5fD4dO3ZMo0eP7nS73++Xx+MJTiclJcnv9/eprL5p/UaTXptkelYAuBQZTxghWe4lU1ZtbW0qKirSPffco9jY2AEtw+v1yuv1SpIKCwvl8Xj0o2d/1O3YyamT9XXT13p06qPyxHq6HRMOUVFRCgQCEbv/viKnuchpHjtklOyTM1QuibJqb29XUVGRpk2bpilTpnSZ73a7VV9fH5xuaGiQ2+3uMi4nJ0c5OTnB6azns/R189eSpPfuek+f1n+qOT+YoxhXTAjWYmA8Hk+ndbMqcpqLnOaxQ0bJPjlDxfanrhuGoeeee07Dhw/X7bff3u2Y7OxslZWVyTAMVVVVKTY2tk+HAA80HAj+nJGQobmj51qqqADgcmH7PatDhw6prKxM6enpWrVqlSRp4cKFwXcgs2fP1uTJk1VRUaHly5crOjpaubm5/b6fKIftex0AbMv2ZXX11Vdr69atFxzjcDi0dOnSMCUCAJiN3YU+OPiLg5GOAACXNcqqF+Pc4xQfHR/pGABwWaOserHt9m2RjgAAlz3bf2YVSlX3VGnooKGRjgEAlz32rC6AogIAa6CsAACWR1kBACyPsgIAWB5lBQCwPMoKAGB5lBUAwPIoKwCA5VFWAADLo6wAAJZHWQEALI+yAgBYHmUFALA8ygoAYHmUFQDA8igrAIDlUVYAAMujrAAAlkdZAQAsj7ICAFgeZQUAsDzKCgBgeZQVAMDyKCsAgOW5Ih3gYm3YsEEVFRVKSEhQUVFRl/kHDhzQ008/reTkZEnSlClTNH/+/HDHBABcBNuX1YwZMzRnzhyVlpb2OGbcuHEqKCgIYyoAgJlsfxhw/PjxiouLi3QMAEAI2X7Pqi+qqqq0atUqJSYmatGiRRo5cmS347xer7xerySpsLBQHo8nnDEHxOVykdNE5DSXHXLaIaNkn5yh4jAMw4h0iIvl8/m0du3abj+zam1tVVRUlGJiYlRRUaGXX35Z69ev79Nya2trzY5qOo/Ho/r6+kjH6BU5zUVO89gho2SfnGlpaSFZru0PA/YmNjZWMTExkqSsrCx1dHSoqakpwqkAAP1xyZfVyZMn9e3OY3V1tQKBgOLj4yOcCgDQH7b/zGrdunX67LPP1NzcrPvvv18LFixQe3u7JGn27NnavXu33n33XTmdTkVHRys/P18OhyPCqQEA/WH7ssrPz7/g/Dlz5mjOnDlhSgMACIVL/jAgAMD+KCsAgOVRVgAAy6OsAACWR1kBACyPsgIAWB5lBQCwPMoKAGB5lBUAwPJCUlZ33313KBYLALhMsWcFALC8fpdVe3u7ampq1NbWFoo8AAB00e8L2T711FOKiYlRU1OTZsyYob///e+Ki4vT0qVLlZ6eHoqMAIDLXL/L6uzZs3r88cfV1tamFStWqLi4WE1NTdq4caN+85vfhCIjAOAy1++ycjgcMgxDMTExSktLU2xsrGJjYxUIBEKRDwCA/pdVYmKiCgoKFAgE1NjYqDfeeENXXXWVWltbQ5EPAID+l9VDDz0k6fyJFrW1tfriiy9UXV2txMRE08MBACD1saxaWlrU0tKi1NTU//2iy6X09HSlpaXppptuCllAAAB6LasdO3bohRdeUCAQUGZmplavXi2n06kPP/xQ5eXl2r9/v1599dVwZAUAXKZ6Latt27YpLy9P48eP1+uvv66SkhJVVVXJ4/Fo8uTJuu2228KREwBwGeu1rJqamoKH+RYvXqwlS5Zo5cqVmjJlSsjDAQAg9eEKFlFR/xsydOhQxcTEUFQAgLDqdc+qra1N9957r0aNGqWMjAwZhiGfz6fk5ORw5AMAoPeyeumll1RTU6OamhodO3ZMKSkpWrFihaKjozVy5Eilp6dr2bJl4cgKALhM9VpWQ4cO1YQJEzRhwoTgbe3t7friiy+CJQYAQCj1+0vB0vnvWGVkZCgjI8PsPAAAdMHfswIAWN6A9qysZsOGDaqoqFBCQoKKioq6zDcMQ5s2bdK+ffs0ePBg5ebmslcIADZySexZzZgxQ6tXr+5x/r59+3T8+HGtX79ey5Yt04svvhjGdACAizWgslq7dm2fxtXX1+ujjz5SbW1tl3nvv//+QO66W+PHj1dcXFyP8/fs2aPp06fL4XBo7NixOn36tBobG027fwBAaA3oMODBgwd7HVNZWani4mIlJyerrq5OM2bM0JIlS4JfMn7hhRfCdgFcv98vj8cTnE5KSpLf7+9ypXiv1yuv1ytJKiws7PQ7VuVyuchpInKayw457ZBRsk/OUAnZZ1Z//etftWLFCmVlZenkyZMqKSnR008/rYcfflgul0uGYYTqrgcsJydHOTk5wen6+voIpukbj8dDThOR01x2yGmHjJJ9cqalpYVkuX0qqzfeeKPT9Llz5zrdNn/+/C6/c/z4cWVlZUmShg0bptWrV6ukpERPPfWUHnnkkYvJ3G9ut7vTg9zQ0CC32x3WDACAgevTZ1aGYVzwX3fi4uI6FYTT6dSKFSuUlJSkJ598UoFAwJw16IPs7GyVlZXJMAxVVVUpNjaWPxYJADbSpz2ru+66q9P0P/7xjy63fd8111yjnTt3dtrrcjgcys3N1fPPP6/Dhw8PIG731q1bp88++0zNzc26//77tWDBArW3t0uSZs+ercmTJ6uiokLLly9XdHS0cnNzTbtvAEDohewzq6VLl6qjo6PbecuWLdO8efNMu6/8/PwLznc4HFq6dKlp9wcACK8BldUPf/jD3hfscsnl6nnxl/NZLQCA/hnQ96wKCgrMzgEAQI8uiStYAAAubZQVAMDyTCur06dPm7UoAAA66fcJFqdOndJXX30V/FdbW6vTp09r8ODB+sMf/hCKjACAy1y/y+q+++7T9OnTNXHiRN1yyy167bXXVFhYGIpsAABIGsBhwLVr1wb/rH1ycrKcTmcocgEAENTvPaurrrpKy5cvV01NjV5++WWdOHFCra2tio2NDUU+AAAGfgWLH/zgB8rLy9PRo0f1/PPPa/DgwfrVr35lZjYAACT1saxaWlrU0tKi1NTULvPS09OVn5+vo0ePmh4OAACpD2W1Y8cOvfDCCwoEAsrMzNTq1avldDr14Ycfqry8XPv379err76qjIyMcOQFAFyGei2rbdu2KS8vT+PHj9frr7+ukpISVVVVyePxaPLkybrtttvCkRMAcBnrtayampqCf35+8eLFWrJkiVauXKkpU6aEPBwAAFIfTl2PivrfkKFDhyomJoaiAgCEVa97Vm1tbbr33ns1atQoZWRkyDAM+Xw+JScnhyMfAAC9l9VLL72kmpoa1dTU6NixY0pJSdGKFSsUHR2tkSNHKj09XcuWLQtHVgDAZarXsho6dKgmTJigCRMmBG/79goW35YYAAChNKAvBbtcLmVkZPR4uvqWLVsuKhQAAN/F37MCAFgeZQUAsDzKCgBgeZQVAMDyKCsAgOVRVgAAy6OsAACWR1kBACxvwH8p2EoqKyu1adMmBQIBzZo1S3Pnzu00f+fOndq8ebPcbrckac6cOZo1a1YkogIABsD2ZRUIBLRx40b99re/VVJSkh577DFlZ2drxIgRncZNnTpVv/zlLyOUEgBwMWx/GLC6ulqpqalKSUmRy+XS1KlTVV5eHulYAAAT2X7Pyu/3KykpKTidlJSkw4cPdxn34Ycf6vPPP9eVV16pX/ziF/J4PF3GeL1eeb1eSVJhYWG3Y6zG5XKR00TkNJcdctoho2SfnKFi+7Lqi+uuu0433nijBg0apH/9618qLS3VE0880WVcTk6OcnJygtP19fXhjDkgHo+HnCYip7nskNMOGSX75ExLSwvJcm1/GNDtdquhoSE43dDQEDyR4lvx8fEaNGiQJGnWrFk6evRoWDMCAC6O7csqMzNTdXV18vl8am9v165du5Sdnd1pTGNjY/DnPXv2dDn5AgBgbbY/DOh0OrVkyRKtWbNGgUBAM2fO1MiRI7VlyxZlZmYqOztb//znP7Vnzx45nU7FxcUpNzc30rEBAP3gMAzDiHQIq6qtrY10hF7Z5Tg2Oc1FTvPYIaNkn5x8ZgUAuGxRVgAAy6OsAACWR1kBACyPsgIAWB5lBQCwPMoKAGB5lBUAwPIoKwCA5VFWAADLo6wAAJZHWQEALI+yAgBYHmUFALA8ygoAYHmUFQDA8igrAIDlUVYAAMujrAAAlkdZAQAsj7ICAFgeZQUAsDzKCgBgeZQVAMDyKCsAgOVRVgAAy6OsAACW54p0ADNUVlZq06ZNCgQCmjVrlubOndtp/rlz5/Tss8/q6NGjio+PV35+vpKTkyOUFgDQX7bfswoEAtq4caNWr16t4uJiffDBB/rqq686jdmxY4eGDh2qkpIS3XbbbXrttdcilBYAMBC2L6vq6mqlpqYqJSVFLpdLU6dOVXl5eacxe/bs0YwZMyRJN9xwgz799FMZhhGBtACAgbB9Wfn9fiUlJQWnk5KS5Pf7exzjdDoVGxur5ubmsOYEAAzcJfGZlVm8Xq+8Xq8kqbCwUB6PJ8KJeudyuchpInKayw457ZBRsk/OULF9WbndbjU0NASnGxoa5Ha7ux2TlJSkjo4Otba2Kj4+vsuycnJylJOTE5yur68PXXCTeDwecpqInOayQ047ZJTskzMtLS0ky7X9YcDMzEzV1dXJ5/Opvb1du3btUnZ2dqcx1113nXbu3ClJ2r17tyZMmCCHwxGBtACAgbD9npXT6dSSJUu0Zs0aBQIBzZw5UyNHjtSWLVuUmZmp7Oxs3XLLLXr22Wf14IMPKi4uTvn5+ZGODQDoB9uXlSRlZWUpKyur021333138Ofo6GitXLky3LEAACax/WFAAMClj7ICAFgeZQUAsDzKCgBgeZQVAMDyKCsAgOVRVgAAy6OsAACWR1kBACyPsgIAWB5lBQCwPMoKAGB5lBUAwPIoKwCA5VFWAADLo6wAAJZHWQEALI+yAgBYHmUFALA8ygoAYHmUFQDA8igrAIDlUVYAAMujrAAAlkdZAQAsj7ICAFgeZQUAsDzKCgBgea5IB7gYLS0tKi4u1jfffKMrrrhCDz30kOLi4rqMu/vuu5Weni5J8ng8evTRR8MdFQBwEWxdVm+99ZauueYazZ07V2+99Zbeeust/exnP+syLjo6Ws8880wEEgIAzGDrw4Dl5eW6+eabJUk333yzysvLI5wIABAKtt6zOnXqlBITEyVJw4YN06lTp7odd+7cORUUFMjpdOrOO+/Uj370o27Heb1eeb1eSVJhYaHS0tJCE9xk5DQXOc1lh5x2yCjZJ2coWH7P6sknn9Svf/3rLv++vxflcDjkcDi6XcaGDRtUWFio5cuX65VXXtHx48e7HZeTk6PCwkIVFhaqoKDA9HUJBXKai5zmskNOO2SUyGn5PavHH3+8x3kJCQlqbGxUYmKiGhsb9X//93/djnO73ZKklJQUjR8/XjU1NUpNTQ1JXgCA+Sy/Z3Uh2dnZeu+99yRJ7733nq6//vouY1paWnTu3DlJUlNTkw4dOqQRI0aENScA4OI4f//73/8+0iEGKiMjQ2+//ba2bdumlpYWLV68WNHR0Tpy5Ii2bt2q7OxsHTt2TGvXrpXX69XOnTt1xx136Nprr+3z8u2AnOYip7nskNMOGaXLO6fDMAzD9KUCAGAiWx8GBABcHigrAIDlWf5swEiorKzUpk2bFAgENGvWLM2dOzds911fX6/S0lKdPHlSDodDOTk5uvXWW7V161b9+9//Dp7xuHDhQmVlZUmStm/frh07digqKkqLFy/WpEmTwrIeeXl5iomJUVRUlJxOpwoLC3u8BJZhGNq0aZP27dunwYMHKzc3N3hce+fOnXrzzTclSfPmzdOMGTNMy1hbW6vi4uLgtM/n04IFC3T69OmIb88NGzaooqJCCQkJKioqktTzJcQGsv2OHj2q0tJSnT17VpMnT9bixYt7/HpHf3Nu3rxZe/fulcvlUkpKinJzczV06FD5fD499NBDwe8DjRkzRsuWLbtgnr5eNm0gOc38f+Pz+bRu3To1NzcrIyNDDz74oFyu/r+EdpezuLhYtbW1kqTW1lbFxsbqmWeeidj27Ol1KKLPTwOddHR0GA888IBx/Phx49y5c8bDDz9sfPnll2G7f7/fbxw5csQwDMNobW01li9fbnz55ZfGli1bjLfffrvL+C+//NJ4+OGHjbNnzxonTpwwHnjgAaOjoyMs65Gbm2ucOnWq022bN282tm/fbhiGYWzfvt3YvHmzYRiGsXfvXmPNmjVGIBAwDh06ZDz22GOGYRhGc3OzkZeXZzQ3N3f6ORQ6OjqMpUuXGj6fzxLb88CBA8aRI0eMlStXBm8zc/sVFBQYhw4dMgKBgLFmzRqjoqLCtJyVlZVGe3t7MPO3OU+cONFp3Hf1lKendTYjp5mPc1FRkfH+++8bhmEYf/nLX4x33nnHtJzf9corrxh/+9vfDMOI3Pbs6XUoks9PDgN+T3V1tVJTU5WSkiKXy6WpU6eG9TJOiYmJwXckQ4YM0fDhw+X3+3scX15erqlTp2rQoEFKTk5WamqqqqurI7YePV0Ca8+ePZo+fbocDofGjh2r06dPq7GxUZWVlZo4caLi4uIUFxeniRMnqrKyMiTZ9u/fr9TUVF1xxRUXzB+u7Tl+/Pgu73jN2n6NjY06c+aMxo4dK4fDoenTpw84b3c5r732WjmdTknS2LFjL/gclXTBPGZdNq27nD3p7+NsGIYOHDigG264QZI0Y8aMkOQ0DEP//e9/deONN15wGaHenj29DkXy+clhwO/x+/1KSkoKTiclJenw4cMRyeLz+XTs2DGNHj1aBw8e1DvvvKOysjJlZGTo5z//ueLi4uT3+zVmzJjg77jd7uALRzjWY82aNZKkH//4x8rJyenxElh+v18ej6dTHr/f32V7fze/2T744INOLwJW3J5mbb/unseh2q47duzQ1KlTg9M+n0+PPPKIhgwZop/85CcaN27cBfP09bJpA2XG49zc3KzY2NhgQYfqefr5558rISFBV155ZfC2SG/P774ORfL5SVlZVFtbm4qKinTPPfcoNjZWs2fP1vz58yVJW7Zs0auvvqrc3NyIZnzyySfldrt16tQp/fGPf+xy3bILXQIr3Nrb27V371799Kc/lSRLbs/vs9L268mbb74pp9OpadOmSTr/jnzDhg2Kj4/X0aNH9cwzzwQ/l+kLs9fZDo/zd33/DVWkt+f3X4fMXHZ/cRjwe9xutxoaGoLTDQ0Nwcs1hUt7e7uKioo0bdo0TZkyRdL5dzFRUVGKiorSrFmzdOTIkW7z+v1+ud3usKzHt8tLSEjQ9ddfr+rq6uAlsCR1ugSW2+1WfX19lzw95Tfbvn37NGrUKA0bNkySNbenJNO2Xzjy7ty5U3v37tXy5cuDL1qDBg1SfHy8pPNfDE1JSVFdXd0F8/S0zmYw63GOj49Xa2urOjo6Oo03U0dHhz766KNOe6mR3J7dvQ5F8vlJWX1PZmam6urq5PP51N7erl27dik7Ozts928Yhp577jkNHz5ct99+e/D2b58gkvTRRx9p5MiRks5fcmrXrl06d+6cfD6f6urqNHr06JCvR1tbm86cORP8+ZNPPlF6enqPl8DKzs5WWVmZDMNQVVWVYmNjlZiYqEmTJunjjz9WS0uLWlpa9PHHHwfPyjLT99+xWm17fsus7ZeYmKghQ4aoqqpKhmGorKzM1LyVlZV6++239eijj2rw4MHB25uamhQIBCRJJ06cUF1dnVJSUi6Ypy+XTRsosx5nh8OhCRMmaPfu3ZLOF7XZj//+/fuVlpbW6fBYpLZnT69DkXx+cgWLblRUVOiVV15RIBDQzJkzNW/evLDd98GDB/W73/1O6enpwXerCxcu1AcffKCamho5HA5dccUVWrZsWfDY8Ztvvqn//Oc/ioqK0j333KPJkyeHfD1OnDihP//5z5LOvyO86aabNG/ePDU3N6u4uFj19fVdTm3duHGjPv74Y0VHRys3N1eZmZmSzn/msX37dknnT22dOXOmaTml82Wam5urZ599Nngoo6SkJOLbc926dfrss8/U3NyshIQELViwQNdff71p2+/IkSPasGGDzp49q0mTJmnJkiUDOmzTXc7t27ervb09eKLAt6dU7969W1u3bpXT6VRUVJTuuuuu4ItQT3l6es6YkfPAgQOmPc4nTpzQunXr1NLSolGjRunBBx/UoEGDTMl5yy23qLS0VGPGjNHs2bODYyO1PXt6HRozZkzEnp+UFQDA8jgMCACwPMoKAGB5lBUAwPIoKwCA5VFWAADLo6wAAJZHWQE2k5eXp08++STSMYCwoqwAAJbHl4IBGykpKdH7778vl8ulqKgozZ8/X3feeWekYwEhR1kBNpOXl6f77rtPEydOjHQUIGw4DAgAsDzKCgBgeZQVAMDyKCvAZoYNGyafzxfpGEBYcYIFYDPl5eV66aWXdObMGc2bN0933HFHpCMBIUdZAQAsj8OAAADLo6wAAJZHWQEALI+yAgBYHmUFALA8ygoAYHmUFQDA8igrAIDl/T8wH+r/+4NNJAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "social_utility = 0.5*(np.asarray(r0ss).mean(axis=0) + np.asarray(r1ss).mean(axis=0))\n",
    "\n",
    "plt.axis([0, max_steps*n_iter, -0.5, 2.5])\n",
    "plt.plot(moving_average(social_utility, max_steps), 'g')\n",
    "plt.xlabel('t');\n",
    "plt.ylabel('$ \\dfrac{R_A + R_B}{2} $');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[50.        , 21.45584508]])"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "defector.Q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[50.        , 28.91425398]])"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cooperator.Q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}