Merge pull request dennybritz#134 from keithmgould/master

update value estimator only after calculating advantage
aprilffff · Jan 29, 2018 · 5334a6f · 5334a6f
2 parents 2a6fe49 + 30326df
commit 5334a6f
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/PolicyGradient/CliffWalk REINFORCE with Baseline Solution.ipynb b/PolicyGradient/CliffWalk REINFORCE with Baseline Solution.ipynb
@@ -196,11 +196,11 @@
     "        for t, transition in enumerate(episode):\n",
     "            # The return after this timestep\n",
     "            total_return = sum(discount_factor**i * t.reward for i, t in enumerate(episode[t:]))\n",
-    "            # Update our value estimator\n",
-    "            estimator_value.update(transition.state, total_return)\n",
     "            # Calculate baseline/advantage\n",
     "            baseline_value = estimator_value.predict(transition.state)            \n",
     "            advantage = total_return - baseline_value\n",
+    "            # Update our value estimator\n",
+    "            estimator_value.update(transition.state, total_return)\n",
     "            # Update our policy estimator\n",
     "            estimator_policy.update(transition.state, advantage, transition.action)\n",
     "    \n",