diff --git a/aepsych/generators/__init__.py b/aepsych/generators/__init__.py index 8febd891d..008a15e48 100644 --- a/aepsych/generators/__init__.py +++ b/aepsych/generators/__init__.py @@ -16,7 +16,6 @@ from .optimize_acqf_generator import AxOptimizeAcqfGenerator, OptimizeAcqfGenerator from .pairwise_optimize_acqf_generator import PairwiseOptimizeAcqfGenerator from .pairwise_sobol_generator import PairwiseSobolGenerator -from .random_generator import RandomGenerator from .random_generator import AxRandomGenerator, RandomGenerator from .semi_p import IntensityAwareSemiPGenerator from .sobol_generator import AxSobolGenerator, SobolGenerator diff --git a/aepsych/generators/optimize_acqf_generator.py b/aepsych/generators/optimize_acqf_generator.py index c2a7dd095..81ec34b9d 100644 --- a/aepsych/generators/optimize_acqf_generator.py +++ b/aepsych/generators/optimize_acqf_generator.py @@ -22,7 +22,6 @@ from botorch.acquisition import AcquisitionFunction from botorch.acquisition.preference import AnalyticExpectedUtilityOfBestOption from botorch.optim import optimize_acqf -from botorch.utils import draw_sobol_samples logger = getLogger() @@ -47,8 +46,6 @@ def __init__( restarts (int): Number of restarts for acquisition function optimization. samps (int): Number of samples for quasi-random initialization of the acquisition function optimizer. max_gen_time (optional, float): Maximum time (in seconds) to optimize the acquisition function. - This is only loosely followed by scipy's optimizer, so consider using a number about 1/3 or - less of what your true upper bound is. """ if acqf_kwargs is None: @@ -103,56 +100,15 @@ def _gen( logger.info("Starting gen...") starttime = time.time() - if self.max_gen_time is None: - new_candidate, _ = optimize_acqf( - acq_function=acqf, - bounds=torch.tensor(np.c_[model.lb, model.ub]).T.to(train_x), - q=num_points, - num_restarts=self.restarts, - raw_samples=self.samps, - **gen_options, - ) - else: - # figure out how long evaluating a single samp - starttime = time.time() - _ = acqf(train_x[0:num_points, :]) - single_eval_time = time.time() - starttime - - # only a heuristic for total num evals since everything is stochastic, - # but the reasoning is: we initialize with self.samps samps, subsample - # self.restarts from them in proportion to the value of the acqf, and - # run that many optimization. So: - # total_time = single_eval_time * n_eval * restarts + single_eval_time * samps - # and we solve for n_eval - n_eval = int( - (self.max_gen_time - single_eval_time * self.samps) - / (single_eval_time * self.restarts) - ) - if n_eval > 10: - # heuristic, if we can't afford 10 evals per restart, just use quasi-random search - options = {"maxfun": n_eval} - logger.info(f"gen maxfun is {n_eval}") - - new_candidate, _ = optimize_acqf( - acq_function=acqf, - bounds=torch.tensor(np.c_[model.lb, model.ub]).T.to(train_x), - q=num_points, - num_restarts=self.restarts, - raw_samples=self.samps, - options=options, - ) - else: - logger.info(f"gen maxfun is {n_eval}, falling back to random search...") - nsamp = max(int(self.max_gen_time / single_eval_time), 10) - # Generate the points at which to sample - bounds = torch.stack((model.lb, model.ub)) - - X = draw_sobol_samples(bounds=bounds, n=nsamp, q=num_points) - - acqvals = acqf(X) - - best_indx = torch.argmax(acqvals, dim=0) - new_candidate = X[best_indx] + new_candidate, _ = optimize_acqf( + acq_function=acqf, + bounds=torch.tensor(np.c_[model.lb, model.ub]).T.to(train_x), + q=num_points, + num_restarts=self.restarts, + raw_samples=self.samps, + timeout_sec=self.max_gen_time, + **gen_options, + ) logger.info(f"Gen done, time={time.time()-starttime}") return new_candidate @@ -258,6 +214,14 @@ class MissingValue: @classmethod def _get_gen_options(cls, config: Config): classname = "OptimizeAcqfGenerator" - restarts = config.getint(classname, "restarts", fallback=10) - samps = config.getint(classname, "samps", fallback=1000) - return {"restarts": restarts, "samps": samps} + restarts = config.getint(classname, "num_restarts", fallback=10) + samps = config.getint(classname, "raw_samples", fallback=1024) + timeout_sec = config.getfloat(classname, "max_gen_time", fallback=None) + optimizer_kwargs = { + "optimizer_kwargs": { + "num_restarts": restarts, + "raw_samples": samps, + "timeout_sec": timeout_sec, + } + } + return {"model_gen_options": optimizer_kwargs} diff --git a/aepsych/models/base.py b/aepsych/models/base.py index 1fb9027b6..5d896c7bc 100644 --- a/aepsych/models/base.py +++ b/aepsych/models/base.py @@ -17,17 +17,17 @@ from aepsych.config import Config, ConfigurableMixin from aepsych.factory.factory import default_mean_covar_factory -from aepsych.models.utils import get_extremum -from aepsych.utils import dim_grid, get_jnd_multid, make_scaled_sobol, promote_0d +from aepsych.models.utils import get_extremum, inv_query +from aepsych.utils import dim_grid, get_jnd_multid, promote_0d from aepsych.utils_logging import getLogger from botorch.fit import fit_gpytorch_mll, fit_gpytorch_mll_scipy from botorch.models.gpytorch import GPyTorchModel from botorch.posteriors import GPyTorchPosterior from gpytorch.likelihoods import Likelihood from gpytorch.mlls import MarginalLogLikelihood -from scipy.optimize import minimize from scipy.stats import norm + logger = getLogger() torch.set_default_dtype(torch.double) # TODO: find a better way to prevent type errors @@ -120,6 +120,7 @@ def get_max( self: ModelProtocol, locked_dims: Optional[Mapping[int, List[float]]] = None, n_samples: int = 1000, + max_time: Optional[float] = None, ) -> Tuple[float, np.ndarray]: """Return the maximum of the modeled function, subject to constraints Returns: @@ -129,12 +130,15 @@ def get_max( n_samples int: number of coarse grid points to sample for optimization estimate. """ locked_dims = locked_dims or {} - return get_extremum(self, "max", self.bounds, locked_dims, n_samples) + return get_extremum( + self, "max", self.bounds, locked_dims, n_samples, max_time=max_time + ) def get_min( self: ModelProtocol, locked_dims: Optional[Mapping[int, List[float]]] = None, n_samples: int = 1000, + max_time: Optional[float] = None, ) -> Tuple[float, np.ndarray]: """Return the minimum of the modeled function, subject to constraints Returns: @@ -144,15 +148,18 @@ def get_min( n_samples int: number of coarse grid points to sample for optimization estimate. """ locked_dims = locked_dims or {} - return get_extremum(self, "min", self.bounds, locked_dims, n_samples) + return get_extremum( + self, "min", self.bounds, locked_dims, n_samples, max_time=max_time + ) def inv_query( - self: ModelProtocol, + self, y: float, locked_dims: Optional[Mapping[int, List[float]]] = None, probability_space: bool = False, n_samples: int = 1000, - ) -> Tuple[float, torch.Tensor]: + max_time: Optional[float] = None, + ) -> Tuple[float, Union[torch.Tensor, np.ndarray]]: """Query the model inverse. Return nearest x such that f(x) = queried y, and also return the value of f at that point. @@ -160,61 +167,27 @@ def inv_query( y (float): Points at which to find the inverse. locked_dims (Mapping[int, List[float]]): Dimensions to fix, so that the inverse is along a slice of the full surface. - probability_space (bool, optional): Is y (and therefore the + probability_space (bool): Is y (and therefore the returned nearest_y) in probability space instead of latent function space? Defaults to False. Returns: Tuple[float, np.ndarray]: Tuple containing the value of f nearest to queried y and the x position of this value. """ - if probability_space: - assert ( - self.outcome_type == "binary" - ), f"Cannot get probability space for outcome_type '{self.outcome_type}'" - - locked_dims = locked_dims or {} - - def model_distance(x, pt, probability_space): - return np.abs( - self.predict(torch.tensor([x]), probability_space=probability_space)[0] - .detach() - .numpy() - - pt - ) - - # Look for point with value closest to y, subject the dict of locked dims - - query_lb = self.lb.clone() - query_ub = self.ub.clone() - - for locked_dim in locked_dims.keys(): - dim_values = locked_dims[locked_dim] - if len(dim_values) == 1: - query_lb[locked_dim] = dim_values[0] - query_ub[locked_dim] = dim_values[0] - else: - query_lb[locked_dim] = dim_values[0] - query_ub[locked_dim] = dim_values[1] - - d = make_scaled_sobol(query_lb, query_ub, n_samples, seed=0) - - bounds = zip(query_lb.numpy(), query_ub.numpy()) - - fmean, _ = self.predict(d, probability_space=probability_space) - - f = torch.abs(fmean - y) - estimate = d[torch.where(f == torch.min(f))[0][0]].numpy() - a = minimize( - model_distance, - estimate, - args=(y, probability_space), - method=self.extremum_solver, - bounds=bounds, + _, arg = inv_query( + self, + y=y, + bounds=self.bounds, + locked_dims=locked_dims, + probability_space=probability_space, + n_samples=n_samples, + max_time=max_time, ) - val = self.predict(torch.tensor([a.x]), probability_space=probability_space)[ - 0 - ].item() - return val, torch.Tensor(a.x) + if probability_space: + val, _ = self.predict_probability(arg.reshape(1, self.dim)) + else: + val, _ = self.predict(arg.reshape(1, self.dim)) + return float(val.item()), arg def get_jnd( self: ModelProtocol, @@ -475,6 +448,7 @@ def get_max( bounds: torch.Tensor, locked_dims: Optional[Mapping[int, List[float]]] = None, n_samples: int = 1000, + max_time: Optional[float] = None, ) -> Tuple[float, np.ndarray]: """Return the maximum of the modeled function, subject to constraints Args: @@ -487,13 +461,16 @@ def get_max( Tuple[torch.Tensor, torch.Tensor]: Tuple containing the max and its location (argmax). """ locked_dims = locked_dims or {} - return get_extremum(self, "max", bounds, locked_dims, n_samples) + return get_extremum( + self, "max", bounds, locked_dims, n_samples, max_time=max_time + ) def get_min( self, bounds: torch.Tensor, locked_dims: Optional[Mapping[int, List[float]]] = None, n_samples: int = 1000, + max_time: Optional[float] = None, ) -> Tuple[float, np.ndarray]: """Return the minimum of the modeled function, subject to constraints Args: @@ -505,7 +482,9 @@ def get_min( Tuple[torch.Tensor, torch.Tensor]: Tuple containing the min and its location (argmin). """ locked_dims = locked_dims or {} - return get_extremum(self, "min", bounds, locked_dims, n_samples) + return get_extremum( + self, "min", bounds, locked_dims, n_samples, max_time=max_time + ) def inv_query( self, @@ -514,7 +493,7 @@ def inv_query( locked_dims: Optional[Mapping[int, List[float]]] = None, probability_space: bool = False, n_samples: int = 1000, - ) -> Tuple[float, torch.Tensor]: + ) -> Tuple[float, Union[torch.Tensor, np.ndarray]]: """Query the model inverse. Return nearest x such that f(x) = queried y, and also return the value of f at that point. @@ -529,51 +508,12 @@ def inv_query( Tuple[float, np.ndarray]: Tuple containing the value of f nearest to queried y and the x position of this value. """ + _, arg = inv_query(self, y, bounds, locked_dims, probability_space, n_samples) if probability_space: - assert ( - self.outcome_type == "binary" or self.outcome_type is None - ), f"Cannot get probability space for outcome_type '{self.outcome_type}'" - pred_function = self.predict_probability - + val, _ = self.predict_probability(arg.reshape(1, -1)) else: - pred_function = self.predict - - locked_dims = locked_dims or {} - - def model_distance(x, pt, probability_space): - return np.abs(pred_function(torch.tensor([x]))[0].detach().numpy() - pt) - - # Look for point with value closest to y, subject the dict of locked dims - - query_lb = bounds[0] - query_ub = bounds[-1] - - for locked_dim in locked_dims.keys(): - dim_values = locked_dims[locked_dim] - if len(dim_values) == 1: - query_lb[locked_dim] = dim_values[0] - query_ub[locked_dim] = dim_values[0] - else: - query_lb[locked_dim] = dim_values[0] - query_ub[locked_dim] = dim_values[1] - - d = make_scaled_sobol(query_lb, query_ub, n_samples, seed=0) - - opt_bounds = zip(query_lb.numpy(), query_ub.numpy()) - - fmean, _ = pred_function(d) - - f = torch.abs(fmean - y) - estimate = d[torch.where(f == torch.min(f))[0][0]].numpy() - a = minimize( - model_distance, - estimate, - args=(y, probability_space), - method=self.extremum_solver, - bounds=opt_bounds, - ) - val = pred_function(torch.tensor([a.x]))[0].item() - return val, torch.Tensor(a.x) + val, _ = self.predict(arg) + return float(val.item()), arg @abc.abstractmethod def get_mll_class(self): diff --git a/aepsych/models/utils.py b/aepsych/models/utils.py index 5b3c0b606..667eb9b09 100644 --- a/aepsych/models/utils.py +++ b/aepsych/models/utils.py @@ -6,20 +6,26 @@ # LICENSE file in the root directory of this source tree. from __future__ import annotations +import warnings + from typing import List, Mapping, Optional, Tuple, Union import numpy as np import torch from botorch.acquisition import PosteriorMean +from botorch.acquisition.objective import PosteriorTransform from botorch.models.model import Model from botorch.models.utils.inducing_point_allocators import GreedyVarianceReduction from botorch.optim import optimize_acqf +from botorch.posteriors import GPyTorchPosterior from botorch.utils.sampling import draw_sobol_samples +from gpytorch.distributions import MultivariateNormal from gpytorch.kernels import Kernel from gpytorch.likelihoods import BernoulliLikelihood from scipy.cluster.vq import kmeans2 from scipy.special import owens_t from scipy.stats import norm +from torch import Tensor from torch.distributions import Normal @@ -104,7 +110,9 @@ def get_probability_space(likelihood, posterior): a_star = fmean / torch.sqrt(1 + fvar) pmean = Normal(0, 1).cdf(a_star) t_term = torch.tensor( - owens_t(a_star.numpy(), 1 / np.sqrt(1 + 2 * fvar.numpy())), + owens_t( + a_star.detach().numpy(), 1 / np.sqrt(1 + 2 * fvar.detach().numpy()) + ), dtype=a_star.dtype, ) pvar = pmean - 2 * t_term - pmean.square() @@ -125,17 +133,27 @@ def get_extremum( bounds: torch.Tensor, locked_dims: Optional[Mapping[int, List[float]]], n_samples: int, + posterior_transform: Optional[PosteriorTransform] = None, + max_time: Optional[float] = None, ) -> Tuple[float, np.ndarray]: """Return the extremum (min or max) of the modeled function Args: - extremum_type (str): type of extremum (currently 'min' or 'max' - n_samples int: number of coarse grid points to sample for optimization estimate. + extremum_type (str): Type of extremum (currently 'min' or 'max'. + bounds (tensor): Lower and upper bounds of the search space. + locked_dims (Mapping[int, List[float]]): Dimensions to fix, so that the + inverse is along a slice of the full surface. + n_samples (int): number of coarse grid points to sample for optimization estimate. + max_time (float): Maximum amount of time in seconds to spend optimizing. Returns: Tuple[float, np.ndarray]: Tuple containing the min and its location (argmin). """ locked_dims = locked_dims or {} - acqf = PosteriorMean(model=model, maximize=(extremum_type == "max")) + acqf = PosteriorMean( + model=model, + maximize=(extremum_type == "max"), + posterior_transform=posterior_transform, + ) best_point, best_val = optimize_acqf( acq_function=acqf, bounds=bounds, @@ -143,9 +161,81 @@ def get_extremum( num_restarts=10, raw_samples=n_samples, fixed_features=locked_dims, + timeout_sec=max_time, ) # PosteriorMean flips the sign on minimize, we flip it back if extremum_type == "min": best_val = -best_val return best_val, best_point.squeeze(0) + + +def inv_query( + model: Model, + y: float, + bounds: torch.Tensor, + locked_dims: Optional[Mapping[int, List[float]]] = None, + probability_space: bool = False, + n_samples: int = 1000, + max_time: Optional[float] = None, +) -> Tuple[float, Union[torch.Tensor, np.ndarray]]: + """Query the model inverse. + Return nearest x such that f(x) = queried y, and also return the + value of f at that point. + Args: + y (float): Points at which to find the inverse. + bounds (tensor): Lower and upper bounds of the search space. + locked_dims (Mapping[int, List[float]]): Dimensions to fix, so that the + inverse is along a slice of the full surface. + probability_space (bool): Is y (and therefore the + returned nearest_y) in probability space instead of latent + function space? Defaults to False. + n_samples (int): number of coarse grid points to sample for optimization estimate. + max_time float: Maximum amount of time in seconds to spend optimizing. + Returns: + Tuple[float, np.ndarray]: Tuple containing the value of f + nearest to queried y and the x position of this value. + """ + locked_dims = locked_dims or {} + if probability_space: + warnings.warn( + "Inverse querying with probability_space=True assumes that the model uses Probit-Bernoulli likelihood!" + ) + posterior_transform = TargetProbabilityDistancePosteriorTransform(y) + else: + posterior_transform = TargetDistancePosteriorTransform(y) + val, arg = get_extremum( + model, "min", bounds, locked_dims, n_samples, posterior_transform, max_time + ) + return val, arg + + +class TargetDistancePosteriorTransform(PosteriorTransform): + def __init__(self, target_value: float): + super().__init__() + self.target_value = target_value + + def evaluate(self, Y: Tensor) -> Tensor: + return (Y - self.target_value) ** 2 + + def _forward(self, mean, var): + q = mean.shape[-2] + batch_shape = mean.shape[:-2] + + new_mean = ((mean - self.target_value) ** 2).view(*batch_shape, q) + mvn = MultivariateNormal(new_mean, var) + return GPyTorchPosterior(mvn) + + def forward(self, posterior: GPyTorchPosterior) -> GPyTorchPosterior: + mean = posterior.mean + var = posterior.variance + return self._forward(mean, var) + + +# Requires botorch approximate model to accept posterior transforms +class TargetProbabilityDistancePosteriorTransform(TargetDistancePosteriorTransform): + def forward(self, posterior: GPyTorchPosterior) -> GPyTorchPosterior: + pmean, pvar = get_probability_space(BernoulliLikelihood(), posterior) + pmean = pmean.unsqueeze(-1).unsqueeze(-1) + pvar = pvar.unsqueeze(-1).unsqueeze(-1) + return self._forward(pmean, pvar) diff --git a/aepsych/models/variational_gp.py b/aepsych/models/variational_gp.py index a0561be6e..c6d38b98e 100644 --- a/aepsych/models/variational_gp.py +++ b/aepsych/models/variational_gp.py @@ -19,7 +19,10 @@ from aepsych.models.ordinal_gp import OrdinalGPModel from aepsych.models.utils import get_probability_space, select_inducing_points from aepsych.utils import get_dim +from botorch.acquisition.objective import PosteriorTransform from botorch.models import SingleTaskVariationalGP + +from botorch.posteriors.gpytorch import GPyTorchPosterior from gpytorch.likelihoods import BernoulliLikelihood, BetaLikelihood from gpytorch.mlls import VariationalELBO @@ -83,6 +86,37 @@ def get_config_options(cls, config: Config, name: Optional[str] = None) -> Dict: return options + def posterior( + self, + X, + output_indices=None, + observation_noise=False, + posterior_transform: Optional[PosteriorTransform] = None, + *args, + **kwargs, + ) -> GPyTorchPosterior: + self.eval() # make sure model is in eval mode + + # input transforms are applied at `posterior` in `eval` mode, and at + # `model.forward()` at the training time + X = self.transform_inputs(X) + + # check for the multi-batch case for multi-outputs b/c this will throw + # warnings + X_ndim = X.ndim + if self.num_outputs > 1 and X_ndim > 2: + X = X.unsqueeze(-3).repeat(*[1] * (X_ndim - 2), self.num_outputs, 1, 1) + dist = self.model(X) + if observation_noise: + dist = self.likelihood(dist, *args, **kwargs) + + posterior = GPyTorchPosterior(distribution=dist) + if hasattr(self, "outcome_transform"): + posterior = self.outcome_transform.untransform_posterior(posterior) + if posterior_transform is not None: + return posterior_transform(posterior) + return posterior + class BinaryClassificationGP(VariationalGP): stimuli_per_trial = 1 diff --git a/aepsych/server/message_handlers/handle_query.py b/aepsych/server/message_handlers/handle_query.py index 332a014a4..4042e0287 100644 --- a/aepsych/server/message_handlers/handle_query.py +++ b/aepsych/server/message_handlers/handle_query.py @@ -30,6 +30,7 @@ def query( x=None, y=None, constraints=None, + max_time=None, ): if server.skip_computations: return None @@ -41,11 +42,11 @@ def query( "constraints": constraints, } if query_type == "max": - fmax, fmax_loc = server.strat.get_max(constraints) + fmax, fmax_loc = server.strat.get_max(constraints, max_time) response["y"] = fmax.item() response["x"] = server._tensor_to_config(fmax_loc) elif query_type == "min": - fmin, fmin_loc = server.strat.get_min(constraints) + fmin, fmin_loc = server.strat.get_min(constraints, max_time) response["y"] = fmin.item() response["x"] = server._tensor_to_config(fmin_loc) elif query_type == "prediction": @@ -62,7 +63,7 @@ def query( # expect constraints to be a dictionary; values are float arrays size 1 (exact) or 2 (upper/lower bnd) constraints = {server.parnames.index(k): v for k, v in constraints.items()} nearest_y, nearest_loc = server.strat.inv_query( - y, constraints, probability_space=probability_space + y, constraints, probability_space=probability_space, max_time=max_time ) response["y"] = nearest_y response["x"] = server._tensor_to_config(nearest_loc) diff --git a/aepsych/strategy.py b/aepsych/strategy.py index 43919eb75..3154f4bf1 100644 --- a/aepsych/strategy.py +++ b/aepsych/strategy.py @@ -228,19 +228,21 @@ def gen(self, num_points: int = 1): return self.generator.gen(num_points, self.model) @ensure_model_is_fresh - def get_max(self, constraints=None): + def get_max(self, constraints=None, max_time=None): constraints = constraints or {} - return self.model.get_max(constraints) + return self.model.get_max(constraints, max_time=max_time) @ensure_model_is_fresh - def get_min(self, constraints=None): + def get_min(self, constraints=None, max_time=None): constraints = constraints or {} - return self.model.get_min(constraints) + return self.model.get_min(constraints, max_time=max_time) @ensure_model_is_fresh - def inv_query(self, y, constraints=None, probability_space=False): + def inv_query(self, y, constraints=None, probability_space=False, max_time=None): constraints = constraints or {} - return self.model.inv_query(y, constraints, probability_space) + return self.model.inv_query( + y, constraints, probability_space, max_time=max_time + ) @ensure_model_is_fresh def predict(self, x, probability_space=False): diff --git a/clients/python/aepsych_client/client.py b/clients/python/aepsych_client/client.py index bf3920539..871fb6540 100644 --- a/clients/python/aepsych_client/client.py +++ b/clients/python/aepsych_client/client.py @@ -233,5 +233,28 @@ def resume(self, config_id: int = None, config_name: str = None): } self._send_recv(request) + def query( + self, + query_type="max", + probability_space=False, + x=None, + y=None, + constraints=None, + max_time=None, + ): + request = { + "type": "query", + "message": { + "query_type": query_type, + "probability_space": probability_space, + "x": x, + "y": y, + "constraints": constraints, + "max_time": max_time, + }, + } + resp = self._send_recv(request) + return resp["y"], resp["x"] + def __del___(self): self.finalize() diff --git a/configs/ax_example.ini b/configs/ax_example.ini index 121bd622b..52e16db70 100644 --- a/configs/ax_example.ini +++ b/configs/ax_example.ini @@ -62,3 +62,6 @@ value = 123 # Value of the fixed parameter, par6. Can be a float or string. [par7] value = placeholder # Value of the fixed parameter, par7. Can be a float or string. + +[OptimizeAcqfGenerator] +max_gen_time = 0.1 diff --git a/configs/single_lse_example.ini b/configs/single_lse_example.ini index ab0f0689d..026f309da 100644 --- a/configs/single_lse_example.ini +++ b/configs/single_lse_example.ini @@ -44,6 +44,10 @@ acqf = EAVC # Use GPClassificationModel or GPRegressionModel for single or PairwiseProbitModel for pairwise. model = GPClassificationModel +max_gen_time = 0.1 # Maximum amount of time to spend optimizing the acquisition function each + # trial. Lower values can lead to faster trials at the expense of possibly + # sampling less informative points. + ## Below this section are configurations of all the classes defined in the section above, ## matching the API in the code. @@ -58,6 +62,9 @@ inducing_size = 100 # previous work). mean_covar_factory = default_mean_covar_factory +max_fit_time = 1.0 # Maximum amount of time to spend fitting the model each trial. Lowering this value + # can lead to faster trials at the expense of model quality. + ## OptimizeAcqfGenerator model settings. [OptimizeAcqfGenerator] diff --git a/tests/generators/test_optimize_acqf_generator.py b/tests/generators/test_optimize_acqf_generator.py index 7d24e93f5..b27504948 100644 --- a/tests/generators/test_optimize_acqf_generator.py +++ b/tests/generators/test_optimize_acqf_generator.py @@ -106,9 +106,9 @@ def test_axoptimizeacqf_config(self): [OptimizeAcqfGenerator] acqf = MCLevelSetEstimation - max_gen_time = 1 - restarts = 1 - samps = 100 + max_gen_time = 0.1 + num_restarts = 1 + raw_samples = 100 [MCLevelSetEstimation] beta = 1 @@ -123,8 +123,24 @@ def test_axoptimizeacqf_config(self): self.assertEqual( gen.model_kwargs["surrogate"].botorch_model_class, ContinuousRegressionGP ) - self.assertEqual(gen.model_gen_kwargs["restarts"], 1) - self.assertEqual(gen.model_gen_kwargs["samps"], 100) + self.assertEqual( + gen.model_gen_kwargs["model_gen_options"]["optimizer_kwargs"][ + "num_restarts" + ], + 1, + ) + self.assertEqual( + gen.model_gen_kwargs["model_gen_options"]["optimizer_kwargs"][ + "raw_samples" + ], + 100, + ) + self.assertEqual( + gen.model_gen_kwargs["model_gen_options"]["optimizer_kwargs"][ + "timeout_sec" + ], + 0.1, + ) self.assertEqual(gen.model_kwargs["acquisition_options"]["target"], 0.5) self.assertEqual(gen.model_kwargs["acquisition_options"]["beta"], 1.0) # TODO: Implement max_gen_time diff --git a/tests/models/test_gp_classification.py b/tests/models/test_gp_classification.py index ca2085e74..6bd334b4a 100644 --- a/tests/models/test_gp_classification.py +++ b/tests/models/test_gp_classification.py @@ -27,14 +27,71 @@ from botorch.acquisition import qUpperConfidenceBound from botorch.optim.fit import fit_gpytorch_mll_torch from botorch.optim.stopping import ExpMAStoppingCriterion -from botorch.posteriors import GPyTorchPosterior -from gpytorch.distributions import MultivariateNormal +from scipy.special import expit from scipy.stats import bernoulli, norm, pearsonr from sklearn.datasets import make_classification from torch.distributions import Normal from torch.optim import Adam -from ..common import cdf_new_novel_det, f_1d, f_2d + +def f_1d(x, mu=0): + """ + latent is just a gaussian bump at mu + """ + return np.exp(-((x - mu) ** 2)) + + +def f_2d(x): + """ + a gaussian bump at 0 , 0 + """ + return np.exp(-np.linalg.norm(x, axis=-1)) + + +def new_novel_det_params(freq, scale_factor=1.0): + """Get the loc and scale params for 2D synthetic novel_det(frequency) function + Keyword arguments: + freq -- 1D array of frequencies whose thresholds to return + scale factor -- scale for the novel_det function, where higher is steeper/lower SD + target -- target threshold + """ + locs = 0.66 * np.power(0.8 * freq * (0.2 * freq - 1), 2) + 0.05 + scale = 2 * locs / (3 * scale_factor) + loc = -1 + 2 * locs + return loc, scale + + +def target_new_novel_det(freq, scale_factor=1.0, target=0.75): + """Get the target (i.e. threshold) for 2D synthetic novel_det(frequency) function + Keyword arguments: + freq -- 1D array of frequencies whose thresholds to return + scale factor -- scale for the novel_det function, where higher is steeper/lower SD + target -- target threshold + """ + locs, scale = new_novel_det_params(freq, scale_factor) + return norm.ppf(target, loc=locs, scale=scale) + + +def new_novel_det(x, scale_factor=1.0): + """Get the cdf for 2D synthetic novel_det(frequency) function + Keyword arguments: + x -- array of shape (n,2) of locations to sample; + x[...,0] is frequency from -1 to 1; x[...,1] is intensity from -1 to 1 + scale factor -- scale for the novel_det function, where higher is steeper/lower SD + """ + freq = x[..., 0] + locs, scale = new_novel_det_params(freq, scale_factor) + return (x[..., 1] - locs) / scale + + +def cdf_new_novel_det(x, scale_factor=1.0): + """Get the cdf for 2D synthetic novel_det(frequency) function + Keyword arguments: + x -- array of shape (n,2) of locations to sample; + x[...,0] is frequency from -1 to 1; x[...,1] is intensity from -1 to 1 + scale factor -- scale for the novel_det function, where higher is steeper/lower SD + """ + return norm.cdf(new_novel_det(x, scale_factor)) class GPClassificationSmoketest(unittest.TestCase): @@ -741,55 +798,6 @@ def test_extra_ask_warns(self): with self.assertWarns(RuntimeWarning): strat.gen() - def test_1d_query(self): - seed = 1 - torch.manual_seed(seed) - np.random.seed(seed) - lb = -4.0 - ub = 4.0 - - strat = Strategy( - lb=lb, - ub=ub, - min_asks=1, - generator=SobolGenerator(lb=lb, ub=ub, seed=seed), - model=GPClassificationModel(lb=lb, ub=ub, inducing_size=50), - stimuli_per_trial=1, - outcome_types=["binary"], - ) - - # mock the posterior call and remove calls that don't need - # to happen - def get_fake_posterior(X, posterior_transform=None): - fmean = torch.sin(torch.pi * X / 4).squeeze(-1) - fcov = torch.eye(fmean.shape[0]) - fake_posterior = GPyTorchPosterior( - mvn=MultivariateNormal(mean=fmean, covariance_matrix=fcov) - ) - return fake_posterior - - strat.model.posterior = get_fake_posterior - strat.model.__call__ = MagicMock() - strat.model.fit = MagicMock() - - x = strat.gen(1) - y = torch.Tensor([1]) - strat.add_data(x, y) - strat.model.set_train_data(x, y) - # We expect the global max to be at (2, 1), the min at (-2, -1) - fmax, argmax = strat.get_max() - self.assertTrue(np.allclose(fmax, 1)) - self.assertTrue(np.allclose(argmax, 2)) - - fmin, argmin = strat.get_min() - self.assertTrue(np.allclose(fmin, -1)) - self.assertTrue(np.allclose(argmin, -2, atol=0.2)) - - # Inverse query at val .85 should return (.85,[2.7]) - val, loc = strat.inv_query(0.85, constraints={}) - self.assertTrue(np.allclose(val, 0.85)) - self.assertTrue(np.allclose(loc.item(), 2.7, atol=1e-2)) - def test_hyperparam_consistency(self): # verify that creating the model `from_config` or with `__init__` has the same hyperparams diff --git a/tests/models/test_model_query.py b/tests/models/test_model_query.py index c528a9c66..687932d90 100644 --- a/tests/models/test_model_query.py +++ b/tests/models/test_model_query.py @@ -9,10 +9,12 @@ import numpy as np import torch -from botorch.fit import fit_gpytorch_mll -from gpytorch.mlls import ExactMarginalLogLikelihood from aepsych.models.exact_gp import ExactGP +from aepsych.models.variational_gp import BinaryClassificationGP +from scipy.special import expit +from scipy.stats import bernoulli + # Fix random seeds np.random.seed(0) @@ -26,8 +28,11 @@ def setUpClass(cls): x = torch.linspace(0.0, 1.0, 10).reshape(-1, 1) y = torch.sin(6.28 * x).reshape(-1, 1) cls.model = ExactGP(x, y) - mll = ExactMarginalLogLikelihood(cls.model.likelihood, cls.model) - fit_gpytorch_mll(mll) + cls.model.fit() + + binary = torch.tensor((-((x - 0.5) ** 2) + 0.05) >= 0, dtype=torch.float64) + cls.binary_model = BinaryClassificationGP(x, binary) + cls.binary_model.fit() def test_min(self): mymin, my_argmin = self.model.get_min(self.bounds) @@ -48,6 +53,22 @@ def test_inverse_query(self): self.assertTrue(-0.01 < val < 0.01) self.assertTrue(0.45 < arg < 0.55) + def test_binary_inverse_query(self): + X = torch.linspace(-3.0, 3.0, 100).reshape(-1, 1) + probs = expit(X) + responses = torch.tensor([float(bernoulli.rvs(p)) for p in probs]).reshape( + -1, 1 + ) + + model = BinaryClassificationGP(X, responses) + model.fit() + + bounds = torch.tensor([[0.0], [1.0]]) + val, arg = model.inv_query(0.75, bounds, probability_space=True) + # Don't need to be precise since we're working with small data. + self.assertTrue(0.7 < val < 0.8) + self.assertTrue(0 < arg < 2) + if __name__ == "__main__": unittest.main()