Feature baseline and evaluation (#50)

* Add stats property to mon

* Small additions in player and PNI logging

* Add damage multiplier method to pokemon

* Add MaxBasePowerPlayer to player.baselines

* Add SimpleHeuristicsPlayer

* Add comment in readme code example

* Add format property to Player

* Add battle against method to Player object

* Add pretty format json pre-commit

* Add evaluate_player function

- Add evaluate function for estimate relative player strength in a comparable way
- Add helper function _estimate_strength_from_result
- Add corresponding _EVALUATION_RATINGS dictionnary containing data used to compute ratings
- Add simple unit tests for edge cases and helper function

* Use math.* instead of np.* in player.utils

* Add edge case unit test for player evaluation (inf value)

* Add unit test for max base power player

* Add Pokemon.damage_multiplier unit test

* Add unit test for SimpleHeuristicPlayer._estimate_matchup

* Add SimpleHeuristicPlayer._should_dynamax unit test

* Expand SimpleHeuristicPlayer unit tests

* Update examples and doc with battle_against method instead of cross_evaluations

.pre-commit-config.yaml

@@ -17,5 +17,6 @@ repos:
       - id: detect-private-key
       - id: fix-encoding-pragma
       - id: mixed-line-ending
+      - id: pretty-format-json
       - id: requirements-txt-fixer
       - id: trailing-whitespace

README.md

@@ -21,6 +21,7 @@ class YourFirstAgent(Player):
                 # A powerful move! Let's use it
                 return self.create_order(move)
 
+        # No available move? Let's switch then!
         for switch in battle.available_switches:
             if switch.current_hp_fraction > battle.active_pokemon.current_hp_fraction:
                 # This other pokemon has more HP left... Let's switch it in?

docs/source/max_damage_player.rst

@@ -82,7 +82,7 @@ We also have to return an order corresponding to a random switch if the player c
 Running and testing our agent
 *****************************
 
-We can now test our agent by crossing evaluating it with a random agent. The complete code is:
+We can now test our agent by making it battle a random agent. The complete code is:
 
 .. code-block:: python
 
@@ -92,7 +92,6 @@ We can now test our agent by crossing evaluating it with a random agent. The com
 
     from poke_env.player.player import Player
     from poke_env.player.random_player import RandomPlayer
-    from poke_env.player.utils import cross_evaluate
 
 
     class MaxDamagePlayer(Player):
@@ -119,18 +118,15 @@ We can now test our agent by crossing evaluating it with a random agent. The com
             battle_format="gen8randombattle",
         )
 
-        # Now, let's evaluate our player
-        cross_evaluation = await cross_evaluate(
-            [random_player, max_damage_player], n_challenges=100
-        )
+    # Now, let's evaluate our player
+    await max_damage_player.battle_against(random_player, n_battles=100)
 
-        print(
-            "Max damage player won %d / 100 battles [this took %f seconds]"
-            % (
-                cross_evaluation[max_damage_player.username][random_player.username] * 100,
-                time.time() - start,
-            )
+    print(
+        "Max damage player won %d / 100 battles [this took %f seconds]"
+        % (
+            max_damage_player.n_won_battles, time.time() - start
         )
+    )
 
 
     if __name__ == "__main__":

docs/source/ou_max_player.rst

@@ -304,7 +304,7 @@ To attribute a team to an agent, you need to pass a ``team`` argument to the age
 Running and testing our agent
 *****************************
 
-We can now test our agent by crossing evaluating it with a random agent. The complete code is:
+We can now test our agent. To do so, we can use the ``cross_evaluate`` function from ``poke_env.player.utils`` or the ``battle_against`` method from ``Player``.
 
 .. code-block:: python
 
@@ -313,7 +313,6 @@ We can now test our agent by crossing evaluating it with a random agent. The com
 
     from poke_env.player.player import Player
     from poke_env.player.random_player import RandomPlayer
-    from poke_env.player.utils import cross_evaluate
 
 
     class MaxDamagePlayer(Player):
@@ -493,13 +492,11 @@ We can now test our agent by crossing evaluating it with a random agent. The com
     )
 
     # Now, let's evaluate our player
-    cross_evaluation = await cross_evaluate(
-        [random_player, max_damage_player], n_challenges=50
-    )
+    await max_damage_player.battle_against(random_player, n_battles = 100)
 
     print(
         "Max damage player won %d / 100 battles"
-        % (cross_evaluation[max_damage_player.username][random_player.username] * 100)
+        % max_damage_player.n_won_battles
     )
 
 

docs/source/using_custom_teambuilder.rst

@@ -192,7 +192,7 @@ Now that we have two players with custom teambuilders, we can make them battle!
 
 .. code-block:: python
 
-    await cross_evaluate([player_1, player_2], n_challenges=5)
+    await player_1.battle_against(player_2, n_battles=5)
 
 The complete example looks like that:
 
@@ -203,7 +203,6 @@ The complete example looks like that:
     import numpy as np
 
     from poke_env.player.random_player import RandomPlayer
-    from poke_env.player.utils import cross_evaluate
     from poke_env.teambuilder.teambuilder import Teambuilder
 
 
@@ -347,10 +346,7 @@ The complete example looks like that:
             max_concurrent_battles=10,
         )
 
-        await cross_evaluate([player_1, player_2], n_challenges=5)
-
-        for battle in player_1.battles:
-            print(battle)
+        await player_1.battle_against(player_2, n_battles=5)
 
 
     if __name__ == "__main__":

examples/custom_teambuilder.py

@@ -3,7 +3,6 @@
 import numpy as np
 
 from poke_env.player.random_player import RandomPlayer
-from poke_env.player.utils import cross_evaluate
 from poke_env.teambuilder.teambuilder import Teambuilder
 
 
@@ -143,10 +142,7 @@ async def main():
         battle_format="gen8ou", team=custom_builder, max_concurrent_battles=10
     )
 
-    await cross_evaluate([player_1, player_2], n_challenges=5)
-
-    for battle in player_1.battles:
-        print(battle)
+    await player_1.battle_against(player_2, n_battles=5)
 
 
 if __name__ == "__main__":

examples/max_damage_player.py

@@ -4,7 +4,6 @@
 
 from poke_env.player.player import Player
 from poke_env.player.random_player import RandomPlayer
-from poke_env.player.utils import cross_evaluate
 
 
 class MaxDamagePlayer(Player):
@@ -28,16 +27,11 @@ async def main():
     max_damage_player = MaxDamagePlayer(battle_format="gen8randombattle")
 
     # Now, let's evaluate our player
-    cross_evaluation = await cross_evaluate(
-        [random_player, max_damage_player], n_challenges=100
-    )
+    await max_damage_player.battle_against(random_player, n_battles=100)
 
     print(
         "Max damage player won %d / 100 battles [this took %f seconds]"
-        % (
-            cross_evaluation[max_damage_player.username][random_player.username] * 100,
-            time.time() - start,
-        )
+        % (max_damage_player.n_won_battles, time.time() - start)
     )
 
 

examples/ou_max_player.py

@@ -4,7 +4,6 @@
 
 from poke_env.player.player import Player
 from poke_env.player.random_player import RandomPlayer
-from poke_env.player.utils import cross_evaluate
 
 
 class MaxDamagePlayer(Player):
@@ -180,14 +179,9 @@ async def main():
     )
 
     # Now, let's evaluate our player
-    cross_evaluation = await cross_evaluate(
-        [random_player, max_damage_player], n_challenges=50
-    )
+    await max_damage_player.battle_against(random_player, n_battles=100)
 
-    print(
-        "Max damage player won %d / 100 battles"
-        % (cross_evaluation[max_damage_player.username][random_player.username] * 100)
-    )
+    print("Max damage player won %d / 100 battles" % max_damage_player.n_won_battles)
 
 
 if __name__ == "__main__":

src/poke_env/environment/pokemon.py

@@ -5,6 +5,7 @@
 from typing import Optional
 from typing import Set
 from typing import Tuple
+from typing import Union
 
 from poke_env.data import POKEDEX
 from poke_env.environment.effect import Effect
@@ -59,6 +60,13 @@ def __init__(
         self._heightm: int
         self._possible_abilities: List[str]
         self._species: str
+        self._stats: Dict[str, Optional[int]] = {
+            "atk": None,
+            "def": None,
+            "spa": None,
+            "spd": None,
+            "spe": None,
+        }
         self._type_1: PokemonType
         self._type_2: Optional[PokemonType] = None
         self._weightkg: int
@@ -359,6 +367,22 @@ def _was_illusionned(self):
         self._status = None
         self._switch_out()
 
+    def damage_multiplier(self, type_or_move: Union[PokemonType, Move]) -> float:
+        """
+        Returns the damage multiplier associated with a given type or move on this
+        pokemon.
+
+        This method is a shortcut for PokemonType.damage_multiplier with relevant types.
+
+        :param type_or_move: The type or move of interest.
+        :type type_or_move: PokemonType or Move
+        :return: The damage multiplier associated with given type on the pokemon.
+        :rtype: float
+        """
+        if isinstance(type_or_move, Move):
+            type_or_move = type_or_move.type
+        return type_or_move.damage_multiplier(self._type_1, self._type_2)
+
     @property
     def ability(self) -> Optional[str]:
         """
@@ -565,6 +589,14 @@ def species(self) -> str:
         """
         return self._species
 
+    @property
+    def stats(self) -> Dict[str, Optional[int]]:
+        """
+        :return: The pokemon's stats, as a dictionary.
+        :rtype: Dict[str, Optional[int]]
+        """
+        return self._stats
+
     @property
     def status(self) -> Optional[Status]:
         """