Warn user if resampling for bounds takes too long in ESs (#462)

## Description

<!-- Provide a brief description of the PR's purpose here. -->

A common error when using bounds is that CMA-ES or another ES can hang
due to resampling, as solutions that fall outside of the bounds need to
be resampled until they are within bounds. This PR adds a warning so
that users will at least know that this behavior is occurring. We are
still unclear how to deal with bounds properly, as it is also an open
research question. #392 has proposed clipping the solutions after a set
number of iterations of resampling but it is unclear if this is the best
solution.

## TODO

<!-- Notable points that this PR has either accomplished or will
accomplish. -->

- [x] Fix slight issue with how OpenAI-ES handles resampling
- [x] Add tests -> since this behavior is supposed to make the tests
hang, we just put this as a script in one of the tests that can be
manually run
- [x] Modify ESs

## Status

- [x] I have read the guidelines in

[CONTRIBUTING.md](https://github.com/icaros-usc/pyribs/blob/master/CONTRIBUTING.md)
- [x] I have formatted my code using `yapf`
- [x] I have tested my code by running `pytest`
- [x] I have linted my code with `pylint`
- [x] I have added a one-line description of my change to the changelog
in
      `HISTORY.md`
- [x] This PR is ready to go

HISTORY.md

@@ -13,6 +13,7 @@
 - Add qd score to lunar lander example ({pr}`458`)
 - Raise error if `result_archive` and `archive` have different fields
   ({pr}`461`)
+- Warn user if resampling for bounds takes too long in ESs ({pr}`462`)
 
 #### Documentation
 

ribs/emitters/opt/_cma_es.py

@@ -3,12 +3,15 @@
 Adapted from Nikolaus Hansen's pycma:
 https://github.com/CMA-ES/pycma/blob/master/cma/purecma.py
 """
+import warnings
+
 import numba as nb
 import numpy as np
 from threadpoolctl import threadpool_limits
 
 from ribs._utils import readonly
-from ribs.emitters.opt._evolution_strategy_base import EvolutionStrategyBase
+from ribs.emitters.opt._evolution_strategy_base import (
+    BOUNDS_SAMPLING_THRESHOLD, BOUNDS_WARNING, EvolutionStrategyBase)
 
 
 class DecompMatrix:
@@ -193,6 +196,7 @@ def ask(self, batch_size=None):
         # Resampling method for bound constraints -> sample new solutions until
         # all solutions are within bounds.
         remaining_indices = np.arange(batch_size)
+        sampling_itrs = 0
         while len(remaining_indices) > 0:
             unscaled_params = self._rng.normal(
                 0.0,
@@ -209,6 +213,11 @@ def ask(self, batch_size=None):
             # out of bounds).
             remaining_indices = remaining_indices[np.any(out_of_bounds, axis=1)]
 
+            # Warn if we have resampled too many times.
+            sampling_itrs += 1
+            if sampling_itrs > BOUNDS_SAMPLING_THRESHOLD:
+                warnings.warn(BOUNDS_WARNING)
+
         return readonly(self._solutions)
 
     def _calc_strat_params(self, num_parents):

ribs/emitters/opt/_evolution_strategy_base.py

@@ -3,6 +3,17 @@
 
 import numpy as np
 
+# Number of times solutions can be resampled before triggering a warning.
+BOUNDS_SAMPLING_THRESHOLD = 100
+
+# Warning for resampling solutions too many times.
+BOUNDS_WARNING = (
+    "During bounds handling, this ES resampled at least "
+    f"{BOUNDS_SAMPLING_THRESHOLD} times. This may indicate that your solution "
+    "space bounds are too tight. When bounds are passed in, the ES resamples "
+    "until all solutions are within the bounds -- if the bounds are too tight "
+    "or the distribution is too large, the ES will resample forever.")
+
 
 class EvolutionStrategyBase(ABC):
     """Base class for evolution strategy optimizers for use with emitters.

ribs/emitters/opt/_lm_ma_es.py

@@ -3,11 +3,14 @@
 Adapted from Nikolaus Hansen's pycma:
 https://github.com/CMA-ES/pycma/blob/master/cma/purecma.py
 """
+import warnings
+
 import numba as nb
 import numpy as np
 
 from ribs._utils import readonly
-from ribs.emitters.opt._evolution_strategy_base import EvolutionStrategyBase
+from ribs.emitters.opt._evolution_strategy_base import (
+    BOUNDS_SAMPLING_THRESHOLD, BOUNDS_WARNING, EvolutionStrategyBase)
 
 
 class LMMAEvolutionStrategy(EvolutionStrategyBase):
@@ -144,6 +147,7 @@ def ask(self, batch_size=None):
         # Resampling method for bound constraints -> sample new solutions until
         # all solutions are within bounds.
         remaining_indices = np.arange(batch_size)
+        sampling_itrs = 0
         while len(remaining_indices) > 0:
             z = self._rng.standard_normal(
                 (len(remaining_indices), self.solution_dim))  # (_, n)
@@ -159,6 +163,11 @@ def ask(self, batch_size=None):
             # out of bounds).
             remaining_indices = remaining_indices[np.any(out_of_bounds, axis=1)]
 
+            # Warn if we have resampled too many times.
+            sampling_itrs += 1
+            if sampling_itrs > BOUNDS_SAMPLING_THRESHOLD:
+                warnings.warn(BOUNDS_WARNING)
+
         return readonly(self._solutions)
 
     @staticmethod

ribs/emitters/opt/_openai_es.py

@@ -2,11 +2,14 @@
 
 See here for more info: https://arxiv.org/abs/1703.03864
 """
+import warnings
+
 import numpy as np
 
 from ribs._utils import readonly
 from ribs.emitters.opt._adam_opt import AdamOpt
-from ribs.emitters.opt._evolution_strategy_base import EvolutionStrategyBase
+from ribs.emitters.opt._evolution_strategy_base import (
+    BOUNDS_SAMPLING_THRESHOLD, BOUNDS_WARNING, EvolutionStrategyBase)
 
 
 class OpenAIEvolutionStrategy(EvolutionStrategyBase):
@@ -58,6 +61,12 @@ def __init__(  # pylint: disable = super-init-not-called
         self._rng = np.random.default_rng(seed)
         self._solutions = None
 
+        if mirror_sampling and not (np.all(lower_bounds == -np.inf) and
+                                    np.all(upper_bounds == np.inf)):
+            raise ValueError("Bounds are currently not supported when using "
+                             "mirror_sampling in OpenAI-ES; see "
+                             "OpenAIEvolutionStrategy.ask() for more info.")
+
         self.mirror_sampling = mirror_sampling
 
         # Default batch size should be an even number for mirror sampling.
@@ -105,14 +114,23 @@ def ask(self, batch_size=None):
         # Resampling method for bound constraints -> sample new solutions until
         # all solutions are within bounds.
         remaining_indices = np.arange(batch_size)
+        sampling_itrs = 0
         while len(remaining_indices) > 0:
             if self.mirror_sampling:
+                # Note that we sample batch_size // 2 here rather than
+                # accounting for len(remaining_indices). This is because we
+                # assume we only run this loop once when mirror_sampling is
+                # True. It is unclear how to do bounds handling when mirror
+                # sampling is involved since the two entries need to be
+                # mirrored. For instance, should we throw out both solutions if
+                # one is out of bounds?
                 noise_half = self._rng.standard_normal(
                     (batch_size // 2, self.solution_dim), dtype=self.dtype)
                 self.noise = np.concatenate((noise_half, -noise_half))
             else:
                 self.noise = self._rng.standard_normal(
-                    (batch_size, self.solution_dim), dtype=self.dtype)
+                    (len(remaining_indices), self.solution_dim),
+                    dtype=self.dtype)
 
             new_solutions = (self.adam_opt.theta[None] +
                              self.sigma0 * self.noise)
@@ -128,6 +146,11 @@ def ask(self, batch_size=None):
             # out of bounds).
             remaining_indices = remaining_indices[np.any(out_of_bounds, axis=1)]
 
+            # Warn if we have resampled too many times.
+            sampling_itrs += 1
+            if sampling_itrs > BOUNDS_SAMPLING_THRESHOLD:
+                warnings.warn(BOUNDS_WARNING)
+
         return readonly(self._solutions)
 
     def tell(self, ranking_indices, ranking_values, num_parents):

ribs/emitters/opt/_sep_cma_es.py

@@ -3,11 +3,14 @@
 Adapted from Nikolaus Hansen's pycma:
 https://github.com/CMA-ES/pycma/blob/master/cma/purecma.py
 """
+import warnings
+
 import numba as nb
 import numpy as np
 
 from ribs._utils import readonly
-from ribs.emitters.opt._evolution_strategy_base import EvolutionStrategyBase
+from ribs.emitters.opt._evolution_strategy_base import (
+    BOUNDS_SAMPLING_THRESHOLD, BOUNDS_WARNING, EvolutionStrategyBase)
 
 
 class DiagonalMatrix:
@@ -148,6 +151,7 @@ def ask(self, batch_size=None):
         # Resampling method for bound constraints -> sample new solutions until
         # all solutions are within bounds.
         remaining_indices = np.arange(batch_size)
+        sampling_itrs = 0
         while len(remaining_indices) > 0:
             unscaled_params = self._rng.normal(
                 0.0,
@@ -164,6 +168,11 @@ def ask(self, batch_size=None):
             # out of bounds).
             remaining_indices = remaining_indices[np.any(out_of_bounds, axis=1)]
 
+            # Warn if we have resampled too many times.
+            sampling_itrs += 1
+            if sampling_itrs > BOUNDS_SAMPLING_THRESHOLD:
+                warnings.warn(BOUNDS_WARNING)
+
         return readonly(self._solutions)
 
     @staticmethod

tests/emitters/evolution_strategy_emitter_test.py

@@ -95,3 +95,39 @@ def test_sphere(es):
         measures_batch = solution_batch[:, :2]
         add_info = archive.add(solution_batch, objective_batch, measures_batch)
         emitter.tell(solution_batch, objective_batch, measures_batch, add_info)
+
+
+if __name__ == "__main__":
+    # For testing bounds handling. Run this file:
+    # python tests/emitters/evolution_strategy_emitter_test.py
+    # The below code should show the resampling warning indicating that the ES
+    # resampled too many times. This test cannot be included in pytest because
+    # it is designed to hang. Comment out the different emitters to test
+    # different ESs.
+
+    archive = GridArchive(solution_dim=31,
+                          dims=[20, 20],
+                          ranges=[(-1.0, 1.0)] * 2)
+    emitter = EvolutionStrategyEmitter(archive,
+                                       x0=np.zeros(31),
+                                       sigma0=1.0,
+                                       bounds=[(0, 1.0)] * 31,
+                                       es="cma_es")
+    #  emitter = EvolutionStrategyEmitter(archive,
+    #                                     x0=np.zeros(31),
+    #                                     sigma0=1.0,
+    #                                     bounds=[(0, 1.0)] * 31,
+    #                                     es="sep_cma_es")
+    #  emitter = EvolutionStrategyEmitter(archive,
+    #                                     x0=np.zeros(31),
+    #                                     sigma0=1.0,
+    #                                     bounds=[(0, 1.0)] * 31,
+    #                                     es="lm_ma_es")
+    #  emitter = EvolutionStrategyEmitter(archive,
+    #                                     x0=np.zeros(31),
+    #                                     sigma0=1.0,
+    #                                     bounds=[(0, 1.0)] * 31,
+    #                                     es="openai_es",
+    #                                     es_kwargs={"mirror_sampling": False})
+
+    emitter.ask()