From d288c2c4ea55b2eb293199339254fce1279e69c8 Mon Sep 17 00:00:00 2001 From: ST John Date: Mon, 9 Dec 2019 10:16:39 +0000 Subject: [PATCH 1/8] upgrade variationally_sparse_gp/models.py to gpflow2 --- .../models/variationally_sparse_gp/models.py | 102 ++++++++++-------- 1 file changed, 60 insertions(+), 42 deletions(-) diff --git a/bayesian_benchmarks/models/variationally_sparse_gp/models.py b/bayesian_benchmarks/models/variationally_sparse_gp/models.py index a5a11d32..f143df4d 100644 --- a/bayesian_benchmarks/models/variationally_sparse_gp/models.py +++ b/bayesian_benchmarks/models/variationally_sparse_gp/models.py @@ -1,9 +1,11 @@ import gpflow import numpy as np +import tensorflow as tf from scipy.cluster.vq import kmeans2 from scipy.stats import norm -class RegressionModel(object): + +class RegressionModel: def __init__(self, is_test=False, seed=0): if is_test: class ARGS: @@ -18,35 +20,43 @@ class ARGS: small_iterations = 1000 initial_likelihood_var = 0.01 self.ARGS = ARGS - self.model = None + self._model = None + self._opt = None + self._model_objective = None def fit(self, X, Y): - if X.shape[0] > self.ARGS.num_inducing: + num_data, input_dim = X.shape + + if num_data > self.ARGS.num_inducing: Z = kmeans2(X, self.ARGS.num_inducing, minit='points')[0] else: # pad with random values - Z = np.concatenate([X, np.random.randn(self.ARGS.num_inducing - X.shape[0], X.shape[1])], 0) + Z = np.concatenate([X, np.random.randn(self.ARGS.num_inducing - num_data, input_dim)], + axis=0) # make model if necessary - if not self.model: - kern = gpflow.kernels.RBF(X.shape[1], lengthscales=float(X.shape[1])**0.5) - lik = gpflow.likelihoods.Gaussian() - lik.variance = self.ARGS.initial_likelihood_var - - self.model = gpflow.models.SGPR(X, Y, kern, feat=Z) - self.model.likelihood.variance = lik.variance.read_value() - self.sess = self.model.enquire_session() - self.opt = gpflow.train.ScipyOptimizer() + if self._model is None: + data = tf.Variable(X, trainable=False), tf.Variable(Y, trainable=False) + kernel = gpflow.kernels.SquaredExponential(lengthscale=float(input_dim)**0.5) + # Gaussian likelihood: use SGPR + self._model = gpflow.models.SGPR(data, kernel, inducing_variable=Z, + noise_variance=self.ARGS.initial_likelihood_var) + @tf.function(autograph=False) + def objective(): + return - self._model.log_marginal_likelihood() + self._model_objective = objective + self._opt = gpflow.optimizers.Scipy() # we might have new data - self.model.X.assign(X, session=self.sess) - self.model.Y.assign(Y, session=self.sess) - self.model.feature.Z.assign(Z, session=self.sess) + self._model.data[0].assign(X) + self._model.data[1].assign(Y) + self._model.inducing_variable.Z.assign(Z) - self.opt.minimize(self.model, session=self.sess, maxiter=self.ARGS.iterations) + self._opt.minimize(self._model_objective, self._model.trainable_variables, + options=dict(maxiter=self.ARGS.iterations)) def predict(self, Xs): - return self.model.predict_y(Xs, session=self.sess) + return self._model.predict_y(Xs) def sample(self, Xs, num_samples): m, v = self.predict(Xs) @@ -55,7 +65,7 @@ def sample(self, Xs, num_samples): return m + np.random.randn(num_samples, N, D) * (v ** 0.5) -class ClassificationModel(object): +class ClassificationModel: def __init__(self, K, is_test=False, seed=0): if is_test: class ARGS: @@ -72,12 +82,19 @@ class ARGS: self.ARGS = ARGS self.K = K - self.model = None + self._model = None + self._model_objective = None + self._opt = None def fit(self, X, Y): - Z = kmeans2(X, self.ARGS.num_inducing, minit='points')[0] if X.shape[0] > self.ARGS.num_inducing else X.copy() + num_data, input_dim = X.shape + + if num_data > self.ARGS.num_inducing: + Z = kmeans2(X, self.ARGS.num_inducing, minit='points')[0] + else: + Z = X.copy() - if not self.model: + if self._model is None: if self.K == 2: lik = gpflow.likelihoods.Bernoulli() num_latent = 1 @@ -85,15 +102,18 @@ def fit(self, X, Y): lik = gpflow.likelihoods.MultiClass(self.K) num_latent = self.K - kern = gpflow.kernels.RBF(X.shape[1], lengthscales=float(X.shape[1]) ** 0.5) - self.model = gpflow.models.SVGP(X, Y, kern, lik, - feat=Z, - whiten=False, - num_latent=num_latent, - minibatch_size=None) + kernel = gpflow.kernels.SquaredExponential(lengthscale=float(input_dim) ** 0.5) + self._model = gpflow.models.SVGP(kernel, lik, + inducing_variable=Z, + whiten=False, + num_latent=num_latent) + + @tf.function(autograph=False) + def objective(X, Y): + return - self._model.log_marginal_likelihood((X, Y)) + self._model_objective = objective - self.sess = self.model.enquire_session() - self.opt = gpflow.train.ScipyOptimizer() + self._opt = gpflow.optimizers.Scipy() iters = self.ARGS.iterations @@ -101,23 +121,21 @@ def fit(self, X, Y): iters = self.ARGS.small_iterations # we might have new data - self.model.X.assign(X, session=self.sess) - self.model.Y.assign(Y, session=self.sess) - self.model.feature.Z.assign(Z, session=self.sess) + self._model.inducing_variable.Z.assign(Z) - num_outputs = self.model.q_sqrt.shape[0] - self.model.q_mu.assign(np.zeros((self.ARGS.num_inducing, num_outputs)), session=self.sess) - self.model.q_sqrt.assign(np.tile(np.eye(self.ARGS.num_inducing)[None], [num_outputs, 1, 1]), session=self.sess) + num_outputs = self._model.q_sqrt.shape[0] + self._model.q_mu.assign(np.zeros((self.ARGS.num_inducing, num_outputs))) + self._model.q_sqrt.assign(np.tile(np.eye(self.ARGS.num_inducing)[None], [num_outputs, 1, 1])) - self.opt.minimize(self.model, maxiter=iters, session=self.sess) + def objective_closure(): + return self._model_objective(tf.constant(X), tf.constant(Y)) + self._opt.minimize(objective_closure, self._model.trainable_variables, + options=dict(maxiter=iters)) def predict(self, Xs): - m, v = self.model.predict_y(Xs, session=self.sess) + m, v = self._model.predict_y(Xs) if self.K == 2: - # convert Bernoulli to onehot + # convert Bernoulli to one-hot return np.concatenate([1 - m, m], 1) else: return m - - - From 0c0e0275957c6d700f36cf342c833753411c1311 Mon Sep 17 00:00:00 2001 From: ST John Date: Mon, 9 Dec 2019 11:02:52 +0000 Subject: [PATCH 2/8] minor cleanup --- .../models/variationally_sparse_gp/models.py | 31 ++++++++++--------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/bayesian_benchmarks/models/variationally_sparse_gp/models.py b/bayesian_benchmarks/models/variationally_sparse_gp/models.py index f143df4d..09d8c9c9 100644 --- a/bayesian_benchmarks/models/variationally_sparse_gp/models.py +++ b/bayesian_benchmarks/models/variationally_sparse_gp/models.py @@ -21,14 +21,13 @@ class ARGS: initial_likelihood_var = 0.01 self.ARGS = ARGS self._model = None - self._opt = None self._model_objective = None def fit(self, X, Y): num_data, input_dim = X.shape if num_data > self.ARGS.num_inducing: - Z = kmeans2(X, self.ARGS.num_inducing, minit='points')[0] + Z, _ = kmeans2(X, self.ARGS.num_inducing, minit='points') else: # pad with random values Z = np.concatenate([X, np.random.randn(self.ARGS.num_inducing - num_data, input_dim)], @@ -36,24 +35,25 @@ def fit(self, X, Y): # make model if necessary if self._model is None: - data = tf.Variable(X, trainable=False), tf.Variable(Y, trainable=False) + data = (tf.Variable(X, trainable=False), tf.Variable(Y, trainable=False)) kernel = gpflow.kernels.SquaredExponential(lengthscale=float(input_dim)**0.5) # Gaussian likelihood: use SGPR self._model = gpflow.models.SGPR(data, kernel, inducing_variable=Z, noise_variance=self.ARGS.initial_likelihood_var) + @tf.function(autograph=False) def objective(): return - self._model.log_marginal_likelihood() self._model_objective = objective - self._opt = gpflow.optimizers.Scipy() # we might have new data self._model.data[0].assign(X) self._model.data[1].assign(Y) self._model.inducing_variable.Z.assign(Z) - self._opt.minimize(self._model_objective, self._model.trainable_variables, - options=dict(maxiter=self.ARGS.iterations)) + opt = gpflow.optimizers.Scipy() + opt.minimize(self._model_objective, self._model.trainable_variables, + options=dict(maxiter=self.ARGS.iterations)) def predict(self, Xs): return self._model.predict_y(Xs) @@ -84,7 +84,6 @@ class ARGS: self.K = K self._model = None self._model_objective = None - self._opt = None def fit(self, X, Y): num_data, input_dim = X.shape @@ -109,12 +108,10 @@ def fit(self, X, Y): num_latent=num_latent) @tf.function(autograph=False) - def objective(X, Y): - return - self._model.log_marginal_likelihood((X, Y)) + def objective(data): + return - self._model.log_marginal_likelihood(data) self._model_objective = objective - self._opt = gpflow.optimizers.Scipy() - iters = self.ARGS.iterations else: @@ -127,15 +124,19 @@ def objective(X, Y): self._model.q_mu.assign(np.zeros((self.ARGS.num_inducing, num_outputs))) self._model.q_sqrt.assign(np.tile(np.eye(self.ARGS.num_inducing)[None], [num_outputs, 1, 1])) + data = (tf.constant(X), tf.constant(Y)) + def objective_closure(): - return self._model_objective(tf.constant(X), tf.constant(Y)) - self._opt.minimize(objective_closure, self._model.trainable_variables, - options=dict(maxiter=iters)) + return self._model_objective(data) + + opt = gpflow.optimizers.Scipy() + opt.minimize(objective_closure, self._model.trainable_variables, + options=dict(maxiter=iters)) def predict(self, Xs): m, v = self._model.predict_y(Xs) if self.K == 2: # convert Bernoulli to one-hot - return np.concatenate([1 - m, m], 1) + return np.concatenate([1 - m, m], axis=1) else: return m From 2b29e0e6ffeec07a6a2265602c527b8465e34535 Mon Sep 17 00:00:00 2001 From: ST John Date: Mon, 9 Dec 2019 11:03:48 +0000 Subject: [PATCH 3/8] upgrade variationally_sparse_gp_minibatch/models.py to gpflow2 --- .../models.py | 129 +++++++++++------- 1 file changed, 79 insertions(+), 50 deletions(-) diff --git a/bayesian_benchmarks/models/variationally_sparse_gp_minibatch/models.py b/bayesian_benchmarks/models/variationally_sparse_gp_minibatch/models.py index eabc7693..c576a8d1 100644 --- a/bayesian_benchmarks/models/variationally_sparse_gp_minibatch/models.py +++ b/bayesian_benchmarks/models/variationally_sparse_gp_minibatch/models.py @@ -1,9 +1,11 @@ import gpflow import numpy as np +import tensorflow as tf from scipy.cluster.vq import kmeans2 from scipy.stats import norm -class RegressionModel(object): + +class RegressionModel: def __init__(self, is_test=False, seed=0): if is_test: class ARGS: @@ -25,29 +27,36 @@ class ARGS: initial_likelihood_var = 0.01 self.ARGS = ARGS self.model = None + self._model = None + self._model_objective = None + self._adam_opt = None + self._natgrad_opt = None def fit(self, X, Y): - if X.shape[0] > self.ARGS.num_inducing: - Z = kmeans2(X, self.ARGS.num_inducing, minit='points')[0] + num_data, input_dim = X.shape + + if num_data > self.ARGS.num_inducing: + Z, _ = kmeans2(X, self.ARGS.num_inducing, minit='points') else: # pad with random values - Z = np.concatenate([X, np.random.randn(self.ARGS.num_inducing - X.shape[0], X.shape[1])], 0) + Z = np.concatenate([X, np.random.randn(self.ARGS.num_inducing - num_data, input_dim)], + axis=0) # make model if necessary if not self.model: - kern = gpflow.kernels.RBF(X.shape[1], lengthscales=float(X.shape[1])**0.5) - lik = gpflow.likelihoods.Gaussian() - lik.variance = self.ARGS.initial_likelihood_var - mb_size = self.ARGS.minibatch_size if X.shape[0] > self.ARGS.minibatch_size else None - self.model = gpflow.models.SVGP(X, Y, kern, lik, feat=Z, minibatch_size=mb_size) + kernel = gpflow.kernels.SquaredExponential(lengthscale=float(input_dim)**0.5) + lik = gpflow.likelihoods.Gaussian(variance=self.ARGS.initial_likelihood_var) + self._model = gpflow.models.SVGP(kernel, likelihood=lik, inducing_variable=Z) - var_list = [[self.model.q_mu, self.model.q_sqrt]] - self.model.q_mu.set_trainable(False) - self.model.q_sqrt.set_trainable(False) - self.ng = gpflow.train.NatGradOptimizer(gamma=self.ARGS.gamma).make_optimize_tensor(self.model, var_list=var_list) - self.adam = gpflow.train.AdamOptimizer(self.ARGS.adam_lr).make_optimize_tensor(self.model) + @tf.function(autograph=False) + def objective(data): + return - self._model.log_marginal_likelihood(data) + self._model_objective = objective - self.sess = self.model.enquire_session() + self._model.q_mu.set_trainable(False) + self._model.q_sqrt.set_trainable(False) + self._natgrad_opt = gpflow.optimizers.NaturalGradient(gamma=self.ARGS.gamma) + self._adam_opt = tf.optimizers.Adam(learning_rate=self.ARGS.adam_lr) iters = self.ARGS.iterations @@ -55,18 +64,27 @@ def fit(self, X, Y): iters = self.ARGS.small_iterations # we might have new data - self.model.X.assign(X, session=self.sess) - self.model.Y.assign(Y, session=self.sess) - self.model.feature.Z.assign(Z, session=self.sess) + self._model.inducing_variable.Z.assign(Z) + num_outputs = self._model.q_sqrt.shape[0] + self._model.q_mu.assign(np.zeros((self.ARGS.num_inducing, num_outputs))) + self._model.q_sqrt.assign(np.tile(np.eye(self.ARGS.num_inducing)[None], [num_outputs, 1, 1])) + + batch_size = np.minimum(self.ARGS.minibatch_size, num_data) + data = (X, Y) + data_minibatch = tf.data.Dataset.from_tensor_slices(data) \ + .prefetch(num_data).repeat().shuffle(num_data) \ + .batch(batch_size) + data_minibatch_it = iter(data_minibatch) - self.model.q_mu.assign(np.zeros((self.ARGS.num_inducing, Y.shape[1])), session=self.sess) - self.model.q_sqrt.assign(np.tile(np.eye(self.ARGS.num_inducing)[None], [Y.shape[1], 1, 1]), session=self.sess) + def objective_closure() -> tf.Tensor: + batch = next(data_minibatch_it) + return self._model_objective(batch) + variational_params = [self._model.q_mu, self._model.q_sqrt] for _ in range(iters): - self.sess.run(self.ng) - self.sess.run(self.adam) - self.model.anchor(session=self.sess) + self._natgrad_opt.minimize(objective_closure, var_list=variational_params) + self._adam_opt.minimize(objective_closure, var_list=self._model.trainable_variables) def predict(self, Xs): return self.model.predict_y(Xs, session=self.sess) @@ -78,7 +96,7 @@ def sample(self, Xs, num_samples): return m + np.random.randn(num_samples, N, D) * (v ** 0.5) -class ClassificationModel(object): +class ClassificationModel: def __init__(self, K, is_test=False, seed=0): if is_test: class ARGS: @@ -97,15 +115,19 @@ class ARGS: self.ARGS = ARGS self.K = K - self.model = None + self._model = None + self._model_objective = None + self._opt = None def fit(self, X, Y): - Z = kmeans2(X, self.ARGS.num_inducing, minit='points')[0] if X.shape[0] > self.ARGS.num_inducing else X.copy() + num_data, input_dim = X.shape - if not self.model: - # NB mb_size does not change once the model is created - mb_size = self.ARGS.minibatch_size if X.shape[0] >= self.ARGS.minibatch_size else None + if num_data > self.ARGS.num_inducing: + Z, _ = kmeans2(X, self.ARGS.num_inducing, minit='points') + else: + Z = X.copy() + if not self.model: if self.K == 2: lik = gpflow.likelihoods.Bernoulli() num_latent = 1 @@ -113,39 +135,46 @@ def fit(self, X, Y): lik = gpflow.likelihoods.MultiClass(self.K) num_latent = self.K - kern = gpflow.kernels.RBF(X.shape[1], lengthscales=float(X.shape[1]) ** 0.5) - self.model = gpflow.models.SVGP(X, Y, kern, lik, - feat=Z, - whiten=False, - num_latent=num_latent, - minibatch_size=mb_size) + kernel = gpflow.kernels.SquaredExponential(lengthscale=float(input_dim)**0.5) + self._model = gpflow.models.SVGP(kernel, likelihood=lik, inducing_variable=Z, + whiten=False, num_latent=num_latent) - self.opt = gpflow.train.AdamOptimizer(self.ARGS.adam_lr) + @tf.function(autograph=False) + def objective(data): + return - self._model.log_marginal_likelihood(data) + self._model_objective = objective + + self._opt = tf.optimizers.Adam(self.ARGS.adam_lr) - self.sess = self.model.enquire_session() iters = self.ARGS.iterations else: iters = self.ARGS.small_iterations # we might have new data - self.model.X.assign(X, session=self.sess) - self.model.Y.assign(Y, session=self.sess) - self.model.feature.Z.assign(Z, session=self.sess) - - num_outputs = self.model.q_sqrt.shape[0] - self.model.q_mu.assign(np.zeros((self.ARGS.num_inducing, num_outputs)), session=self.sess) - self.model.q_sqrt.assign(np.tile(np.eye(self.ARGS.num_inducing)[None], [num_outputs, 1, 1]), session=self.sess) + self._model.inducing_variable.Z.assign(Z) + num_outputs = self._model.q_sqrt.shape[0] + self._model.q_mu.assign(np.zeros((self.ARGS.num_inducing, num_outputs))) + self._model.q_sqrt.assign(np.tile(np.eye(self.ARGS.num_inducing)[None], [num_outputs, 1, 1])) + + batch_size = np.minimum(self.ARGS.minibatch_size, num_data) + data = (X, Y) + data_minibatch = tf.data.Dataset.from_tensor_slices(data) \ + .prefetch(num_data).repeat().shuffle(num_data) \ + .batch(batch_size) + data_minibatch_it = iter(data_minibatch) + + def objective_closure() -> tf.Tensor: + batch = next(data_minibatch_it) + return self._model_objective(batch) - self.opt.minimize(self.model, maxiter=iters, session=self.sess) + for _ in range(iters): + self._opt.minimize(objective_closure, var_list=self._model.trainable_variables) def predict(self, Xs): - m, v = self.model.predict_y(Xs, session=self.sess) + m, v = self.model.predict_y(Xs) if self.K == 2: # convert Bernoulli to onehot - return np.concatenate([1 - m, m], 1) + return np.concatenate([1 - m, m], axis=1) else: return m - - - From afbda2a5429178e45575292c90c757797ce89c93 Mon Sep 17 00:00:00 2001 From: ST John Date: Mon, 9 Dec 2019 11:24:46 +0000 Subject: [PATCH 4/8] bugfixes --- .../models.py | 31 +++++++++++++------ 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/bayesian_benchmarks/models/variationally_sparse_gp_minibatch/models.py b/bayesian_benchmarks/models/variationally_sparse_gp_minibatch/models.py index c576a8d1..5e9c2b72 100644 --- a/bayesian_benchmarks/models/variationally_sparse_gp_minibatch/models.py +++ b/bayesian_benchmarks/models/variationally_sparse_gp_minibatch/models.py @@ -26,7 +26,6 @@ class ARGS: minibatch_size = 1000 initial_likelihood_var = 0.01 self.ARGS = ARGS - self.model = None self._model = None self._model_objective = None self._adam_opt = None @@ -43,7 +42,7 @@ def fit(self, X, Y): axis=0) # make model if necessary - if not self.model: + if self._model is None: kernel = gpflow.kernels.SquaredExponential(lengthscale=float(input_dim)**0.5) lik = gpflow.likelihoods.Gaussian(variance=self.ARGS.initial_likelihood_var) self._model = gpflow.models.SVGP(kernel, likelihood=lik, inducing_variable=Z) @@ -53,8 +52,8 @@ def objective(data): return - self._model.log_marginal_likelihood(data) self._model_objective = objective - self._model.q_mu.set_trainable(False) - self._model.q_sqrt.set_trainable(False) + self._model.q_mu.trainable = False + self._model.q_sqrt.trainable = False self._natgrad_opt = gpflow.optimizers.NaturalGradient(gamma=self.ARGS.gamma) self._adam_opt = tf.optimizers.Adam(learning_rate=self.ARGS.adam_lr) @@ -80,14 +79,22 @@ def objective_closure() -> tf.Tensor: batch = next(data_minibatch_it) return self._model_objective(batch) - variational_params = [self._model.q_mu, self._model.q_sqrt] + variational_params = [(self._model.q_mu, self._model.q_sqrt)] - for _ in range(iters): + @tf.function + def natgrad_step(): self._natgrad_opt.minimize(objective_closure, var_list=variational_params) + + @tf.function + def adam_step(): self._adam_opt.minimize(objective_closure, var_list=self._model.trainable_variables) + for _ in range(iters): + natgrad_step() + adam_step() + def predict(self, Xs): - return self.model.predict_y(Xs, session=self.sess) + return self._model.predict_y(Xs) def sample(self, Xs, num_samples): m, v = self.predict(Xs) @@ -127,7 +134,7 @@ def fit(self, X, Y): else: Z = X.copy() - if not self.model: + if self._model is None: if self.K == 2: lik = gpflow.likelihoods.Bernoulli() num_latent = 1 @@ -168,11 +175,15 @@ def objective_closure() -> tf.Tensor: batch = next(data_minibatch_it) return self._model_objective(batch) - for _ in range(iters): + @tf.function + def adam_step(): self._opt.minimize(objective_closure, var_list=self._model.trainable_variables) + for _ in range(iters): + adam_step() + def predict(self, Xs): - m, v = self.model.predict_y(Xs) + m, v = self._model.predict_y(Xs) if self.K == 2: # convert Bernoulli to onehot return np.concatenate([1 - m, m], axis=1) From 65b1f7d690aef96d8c756d6812d89a04735631e1 Mon Sep 17 00:00:00 2001 From: ST John Date: Mon, 9 Dec 2019 11:25:05 +0000 Subject: [PATCH 5/8] use tqdm if available --- .../models/variationally_sparse_gp_minibatch/models.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/bayesian_benchmarks/models/variationally_sparse_gp_minibatch/models.py b/bayesian_benchmarks/models/variationally_sparse_gp_minibatch/models.py index 5e9c2b72..87dba530 100644 --- a/bayesian_benchmarks/models/variationally_sparse_gp_minibatch/models.py +++ b/bayesian_benchmarks/models/variationally_sparse_gp_minibatch/models.py @@ -4,6 +4,10 @@ from scipy.cluster.vq import kmeans2 from scipy.stats import norm +try: + from tqdm import trange +except ImportError: + trange = range class RegressionModel: def __init__(self, is_test=False, seed=0): @@ -89,7 +93,7 @@ def natgrad_step(): def adam_step(): self._adam_opt.minimize(objective_closure, var_list=self._model.trainable_variables) - for _ in range(iters): + for _ in trange(iters): natgrad_step() adam_step() @@ -179,7 +183,7 @@ def objective_closure() -> tf.Tensor: def adam_step(): self._opt.minimize(objective_closure, var_list=self._model.trainable_variables) - for _ in range(iters): + for _ in trange(iters): adam_step() def predict(self, Xs): From 35d4971bb8c74fde3ffeaca8f65e3a0e1a9f50a2 Mon Sep 17 00:00:00 2001 From: ST John Date: Mon, 26 Sep 2022 13:35:01 +0300 Subject: [PATCH 6/8] update SVGP models for gpflow2 --- .../models/variationally_sparse_gp/models.py | 63 +++++------ .../models.py | 104 ++++++++++-------- 2 files changed, 88 insertions(+), 79 deletions(-) diff --git a/bayesian_benchmarks/models/variationally_sparse_gp/models.py b/bayesian_benchmarks/models/variationally_sparse_gp/models.py index a5a11d32..5946e4e7 100644 --- a/bayesian_benchmarks/models/variationally_sparse_gp/models.py +++ b/bayesian_benchmarks/models/variationally_sparse_gp/models.py @@ -21,38 +21,35 @@ class ARGS: self.model = None def fit(self, X, Y): - if X.shape[0] > self.ARGS.num_inducing: + N, D = X.shape + if N > self.ARGS.num_inducing: Z = kmeans2(X, self.ARGS.num_inducing, minit='points')[0] else: # pad with random values - Z = np.concatenate([X, np.random.randn(self.ARGS.num_inducing - X.shape[0], X.shape[1])], 0) + Z = np.concatenate([X, np.random.randn(self.ARGS.num_inducing - N, D)], 0) # make model if necessary if not self.model: - kern = gpflow.kernels.RBF(X.shape[1], lengthscales=float(X.shape[1])**0.5) - lik = gpflow.likelihoods.Gaussian() - lik.variance = self.ARGS.initial_likelihood_var + lengthscales = np.full(D, float(D)**0.5) + kernel = gpflow.kernels.SquaredExponential(lengthscales=lengthscales) - self.model = gpflow.models.SGPR(X, Y, kern, feat=Z) - self.model.likelihood.variance = lik.variance.read_value() - self.sess = self.model.enquire_session() - self.opt = gpflow.train.ScipyOptimizer() + self.model = gpflow.models.SGPR((X, Y), kernel, inducing_variable=Z, noise_variance=self.ARGS.initial_likelihood_var) + self.opt = gpflow.optimizers.Scipy() # we might have new data - self.model.X.assign(X, session=self.sess) - self.model.Y.assign(Y, session=self.sess) - self.model.feature.Z.assign(Z, session=self.sess) + self.model.data = (X, Y) + self.model.inducing_variable.Z.assign(Z) - self.opt.minimize(self.model, session=self.sess, maxiter=self.ARGS.iterations) + self.opt.minimize(self.model.training_loss, self.model.trainable_variables, options=dict(maxiter=self.ARGS.iterations)) def predict(self, Xs): - return self.model.predict_y(Xs, session=self.sess) + return self.model.predict_y(Xs) def sample(self, Xs, num_samples): m, v = self.predict(Xs) - N, D = np.shape(m) + N, L = np.shape(m) m, v = np.expand_dims(m, 0), np.expand_dims(v, 0) - return m + np.random.randn(num_samples, N, D) * (v ** 0.5) + return m + np.random.randn(num_samples, N, L) * (v ** 0.5) class ClassificationModel(object): @@ -75,7 +72,8 @@ class ARGS: self.model = None def fit(self, X, Y): - Z = kmeans2(X, self.ARGS.num_inducing, minit='points')[0] if X.shape[0] > self.ARGS.num_inducing else X.copy() + N, D = X.shape + Z = kmeans2(X, self.ARGS.num_inducing, minit='points')[0] if N > self.ARGS.num_inducing else X.copy() if not self.model: if self.K == 2: @@ -85,15 +83,15 @@ def fit(self, X, Y): lik = gpflow.likelihoods.MultiClass(self.K) num_latent = self.K - kern = gpflow.kernels.RBF(X.shape[1], lengthscales=float(X.shape[1]) ** 0.5) - self.model = gpflow.models.SVGP(X, Y, kern, lik, - feat=Z, - whiten=False, - num_latent=num_latent, - minibatch_size=None) + lengthscales = np.full(D, float(D)**0.5) + kernel = gpflow.kernels.SquaredExponential(lengthscales=lengthscales) - self.sess = self.model.enquire_session() - self.opt = gpflow.train.ScipyOptimizer() + self.model = gpflow.models.SVGP(kernel, lik, + inducing_variable=Z, + num_latent_gps=num_latent, + num_data=N) + + self.opt = gpflow.optimizers.Scipy() iters = self.ARGS.iterations @@ -101,23 +99,18 @@ def fit(self, X, Y): iters = self.ARGS.small_iterations # we might have new data - self.model.X.assign(X, session=self.sess) - self.model.Y.assign(Y, session=self.sess) - self.model.feature.Z.assign(Z, session=self.sess) + self.model.inducing_variable.Z.assign(Z) num_outputs = self.model.q_sqrt.shape[0] - self.model.q_mu.assign(np.zeros((self.ARGS.num_inducing, num_outputs)), session=self.sess) - self.model.q_sqrt.assign(np.tile(np.eye(self.ARGS.num_inducing)[None], [num_outputs, 1, 1]), session=self.sess) + self.model.q_mu.assign(np.zeros((self.ARGS.num_inducing, num_outputs))) + self.model.q_sqrt.assign(np.tile(np.eye(self.ARGS.num_inducing)[None], [num_outputs, 1, 1])) - self.opt.minimize(self.model, maxiter=iters, session=self.sess) + self.opt.minimize(self.model.training_loss_closure((X, Y)), self.model.trainable_variables, options=dict(maxiter=iters)) def predict(self, Xs): - m, v = self.model.predict_y(Xs, session=self.sess) + m, v = self.model.predict_y(Xs) if self.K == 2: # convert Bernoulli to onehot return np.concatenate([1 - m, m], 1) else: return m - - - diff --git a/bayesian_benchmarks/models/variationally_sparse_gp_minibatch/models.py b/bayesian_benchmarks/models/variationally_sparse_gp_minibatch/models.py index eabc7693..0a33b313 100644 --- a/bayesian_benchmarks/models/variationally_sparse_gp_minibatch/models.py +++ b/bayesian_benchmarks/models/variationally_sparse_gp_minibatch/models.py @@ -1,4 +1,5 @@ import gpflow +import tensorflow as tf import numpy as np from scipy.cluster.vq import kmeans2 from scipy.stats import norm @@ -27,27 +28,27 @@ class ARGS: self.model = None def fit(self, X, Y): - if X.shape[0] > self.ARGS.num_inducing: + N, D = X.shape + _, K = Y.shape + + if N > self.ARGS.num_inducing: Z = kmeans2(X, self.ARGS.num_inducing, minit='points')[0] else: # pad with random values - Z = np.concatenate([X, np.random.randn(self.ARGS.num_inducing - X.shape[0], X.shape[1])], 0) + Z = np.concatenate([X, np.random.randn(self.ARGS.num_inducing - N, D)], 0) # make model if necessary if not self.model: - kern = gpflow.kernels.RBF(X.shape[1], lengthscales=float(X.shape[1])**0.5) - lik = gpflow.likelihoods.Gaussian() - lik.variance = self.ARGS.initial_likelihood_var - mb_size = self.ARGS.minibatch_size if X.shape[0] > self.ARGS.minibatch_size else None - self.model = gpflow.models.SVGP(X, Y, kern, lik, feat=Z, minibatch_size=mb_size) - - var_list = [[self.model.q_mu, self.model.q_sqrt]] - self.model.q_mu.set_trainable(False) - self.model.q_sqrt.set_trainable(False) - self.ng = gpflow.train.NatGradOptimizer(gamma=self.ARGS.gamma).make_optimize_tensor(self.model, var_list=var_list) - self.adam = gpflow.train.AdamOptimizer(self.ARGS.adam_lr).make_optimize_tensor(self.model) + lengthscales = np.full(D, float(D)**0.5) + kernel = gpflow.kernels.SquaredExponential(lengthscales=lengthscales) + lik = gpflow.likelihoods.Gaussian(variance=self.ARGS.initial_likelihood_var) + self.model = gpflow.models.SVGP(kernel, lik, inducing_variable=Z, num_data=N) - self.sess = self.model.enquire_session() + self.variational_params = [(self.model.q_mu, self.model.q_sqrt)] + gpflow.set_trainable(self.model.q_mu, False) + gpflow.set_trainable(self.model.q_sqrt, False) + self.natgrad = gpflow.optimizers.NaturalGradient(gamma=self.ARGS.gamma) + self.adam = tf.optimizers.Adam(self.ARGS.adam_lr) iters = self.ARGS.iterations @@ -55,21 +56,32 @@ def fit(self, X, Y): iters = self.ARGS.small_iterations # we might have new data - self.model.X.assign(X, session=self.sess) - self.model.Y.assign(Y, session=self.sess) - self.model.feature.Z.assign(Z, session=self.sess) + self.model.inducing_variable.Z.assign(Z) - self.model.q_mu.assign(np.zeros((self.ARGS.num_inducing, Y.shape[1])), session=self.sess) - self.model.q_sqrt.assign(np.tile(np.eye(self.ARGS.num_inducing)[None], [Y.shape[1], 1, 1]), session=self.sess) + self.model.q_mu.assign(np.zeros((self.ARGS.num_inducing, K))) + self.model.q_sqrt.assign(np.tile(np.eye(self.ARGS.num_inducing)[None], [K, 1, 1])) + + if N > self.ARGS.minibatch_size: + loss = self.model.training_loss_closure((X, Y)) + else: + train_dataset = tf.data.Dataset.from_tensor_slices((X, Y)).repeat().shuffle(N) + train_iter = iter(train_dataset.batch(self.ARGS.minibatch_size)) + loss = self.model.training_loss_closure(train_iter) + @tf.function + def natgrad_step(): + self.natgrad.minimize(loss, var_list=self.variational_params) + + @tf.function + def adam_step(): + self.adam.minimize(loss, var_list=self.model.trainable_variables) for _ in range(iters): - self.sess.run(self.ng) - self.sess.run(self.adam) - self.model.anchor(session=self.sess) + natgrad_step() + adam_step() def predict(self, Xs): - return self.model.predict_y(Xs, session=self.sess) + return self.model.predict_y(Xs) def sample(self, Xs, num_samples): m, v = self.predict(Xs) @@ -100,12 +112,10 @@ class ARGS: self.model = None def fit(self, X, Y): - Z = kmeans2(X, self.ARGS.num_inducing, minit='points')[0] if X.shape[0] > self.ARGS.num_inducing else X.copy() + N, D = X.shape + Z = kmeans2(X, self.ARGS.num_inducing, minit='points')[0] if N > self.ARGS.num_inducing else X.copy() if not self.model: - # NB mb_size does not change once the model is created - mb_size = self.ARGS.minibatch_size if X.shape[0] >= self.ARGS.minibatch_size else None - if self.K == 2: lik = gpflow.likelihoods.Bernoulli() num_latent = 1 @@ -113,39 +123,45 @@ def fit(self, X, Y): lik = gpflow.likelihoods.MultiClass(self.K) num_latent = self.K - kern = gpflow.kernels.RBF(X.shape[1], lengthscales=float(X.shape[1]) ** 0.5) - self.model = gpflow.models.SVGP(X, Y, kern, lik, + lengthscales = np.full(D, float(D)**0.5) + kernel = gpflow.kernels.RBF(lengthscales=lengthscales) + self.model = gpflow.models.SVGP(kernel, lik, feat=Z, - whiten=False, - num_latent=num_latent, - minibatch_size=mb_size) + num_latent_gps=num_latent, + num_data=N) - self.opt = gpflow.train.AdamOptimizer(self.ARGS.adam_lr) + self.opt = tf.optimizers.Adam(self.ARGS.adam_lr) - self.sess = self.model.enquire_session() iters = self.ARGS.iterations else: iters = self.ARGS.small_iterations # we might have new data - self.model.X.assign(X, session=self.sess) - self.model.Y.assign(Y, session=self.sess) - self.model.feature.Z.assign(Z, session=self.sess) + self.model.inducing_variable.Z.assign(Z) num_outputs = self.model.q_sqrt.shape[0] - self.model.q_mu.assign(np.zeros((self.ARGS.num_inducing, num_outputs)), session=self.sess) - self.model.q_sqrt.assign(np.tile(np.eye(self.ARGS.num_inducing)[None], [num_outputs, 1, 1]), session=self.sess) + self.model.q_mu.assign(np.zeros((self.ARGS.num_inducing, num_outputs))) + self.model.q_sqrt.assign(np.tile(np.eye(self.ARGS.num_inducing)[None], [num_outputs, 1, 1])) - self.opt.minimize(self.model, maxiter=iters, session=self.sess) + if N < self.ARGS.minibatch_size: + loss = self.model.training_loss_closure((X, Y)) + else: + train_dataset = tf.data.Dataset.from_tensor_slices((X, Y)).repeat().shuffle(N) + train_iter = iter(train_dataset.batch(self.ARGS.minibatch_size)) + loss = self.model.training_loss_closure(train_iter) + + @tf.function + def step(): + self.opt.minimize(loss, self.model.trainable_variables) + + for _ in range(iters): + step() def predict(self, Xs): - m, v = self.model.predict_y(Xs, session=self.sess) + m, v = self.model.predict_y(Xs) if self.K == 2: # convert Bernoulli to onehot return np.concatenate([1 - m, m], 1) else: return m - - - From ce074318670f7b9cfe7bf654c962f51454af5123 Mon Sep 17 00:00:00 2001 From: ST John Date: Mon, 26 Sep 2022 15:21:43 +0300 Subject: [PATCH 7/8] _model -> model --- .../models/variationally_sparse_gp/models.py | 48 +++++++-------- .../models.py | 58 +++++++++---------- 2 files changed, 53 insertions(+), 53 deletions(-) diff --git a/bayesian_benchmarks/models/variationally_sparse_gp/models.py b/bayesian_benchmarks/models/variationally_sparse_gp/models.py index 09d8c9c9..0fb9db5c 100644 --- a/bayesian_benchmarks/models/variationally_sparse_gp/models.py +++ b/bayesian_benchmarks/models/variationally_sparse_gp/models.py @@ -20,8 +20,8 @@ class ARGS: small_iterations = 1000 initial_likelihood_var = 0.01 self.ARGS = ARGS - self._model = None - self._model_objective = None + self.model = None + self.model_objective = None def fit(self, X, Y): num_data, input_dim = X.shape @@ -34,29 +34,29 @@ def fit(self, X, Y): axis=0) # make model if necessary - if self._model is None: + if self.model is None: data = (tf.Variable(X, trainable=False), tf.Variable(Y, trainable=False)) kernel = gpflow.kernels.SquaredExponential(lengthscale=float(input_dim)**0.5) # Gaussian likelihood: use SGPR - self._model = gpflow.models.SGPR(data, kernel, inducing_variable=Z, + self.model = gpflow.models.SGPR(data, kernel, inducing_variable=Z, noise_variance=self.ARGS.initial_likelihood_var) @tf.function(autograph=False) def objective(): - return - self._model.log_marginal_likelihood() - self._model_objective = objective + return - self.model.log_marginal_likelihood() + self.model_objective = objective # we might have new data - self._model.data[0].assign(X) - self._model.data[1].assign(Y) - self._model.inducing_variable.Z.assign(Z) + self.model.data[0].assign(X) + self.model.data[1].assign(Y) + self.model.inducing_variable.Z.assign(Z) opt = gpflow.optimizers.Scipy() - opt.minimize(self._model_objective, self._model.trainable_variables, + opt.minimize(self.model_objective, self.model.trainable_variables, options=dict(maxiter=self.ARGS.iterations)) def predict(self, Xs): - return self._model.predict_y(Xs) + return self.model.predict_y(Xs) def sample(self, Xs, num_samples): m, v = self.predict(Xs) @@ -82,8 +82,8 @@ class ARGS: self.ARGS = ARGS self.K = K - self._model = None - self._model_objective = None + self.model = None + self.model_objective = None def fit(self, X, Y): num_data, input_dim = X.shape @@ -93,7 +93,7 @@ def fit(self, X, Y): else: Z = X.copy() - if self._model is None: + if self.model is None: if self.K == 2: lik = gpflow.likelihoods.Bernoulli() num_latent = 1 @@ -102,15 +102,15 @@ def fit(self, X, Y): num_latent = self.K kernel = gpflow.kernels.SquaredExponential(lengthscale=float(input_dim) ** 0.5) - self._model = gpflow.models.SVGP(kernel, lik, + self.model = gpflow.models.SVGP(kernel, lik, inducing_variable=Z, whiten=False, num_latent=num_latent) @tf.function(autograph=False) def objective(data): - return - self._model.log_marginal_likelihood(data) - self._model_objective = objective + return - self.model.log_marginal_likelihood(data) + self.model_objective = objective iters = self.ARGS.iterations @@ -118,23 +118,23 @@ def objective(data): iters = self.ARGS.small_iterations # we might have new data - self._model.inducing_variable.Z.assign(Z) + self.model.inducing_variable.Z.assign(Z) - num_outputs = self._model.q_sqrt.shape[0] - self._model.q_mu.assign(np.zeros((self.ARGS.num_inducing, num_outputs))) - self._model.q_sqrt.assign(np.tile(np.eye(self.ARGS.num_inducing)[None], [num_outputs, 1, 1])) + num_outputs = self.model.q_sqrt.shape[0] + self.model.q_mu.assign(np.zeros((self.ARGS.num_inducing, num_outputs))) + self.model.q_sqrt.assign(np.tile(np.eye(self.ARGS.num_inducing)[None], [num_outputs, 1, 1])) data = (tf.constant(X), tf.constant(Y)) def objective_closure(): - return self._model_objective(data) + return self.model_objective(data) opt = gpflow.optimizers.Scipy() - opt.minimize(objective_closure, self._model.trainable_variables, + opt.minimize(objective_closure, self.model.trainable_variables, options=dict(maxiter=iters)) def predict(self, Xs): - m, v = self._model.predict_y(Xs) + m, v = self.model.predict_y(Xs) if self.K == 2: # convert Bernoulli to one-hot return np.concatenate([1 - m, m], axis=1) diff --git a/bayesian_benchmarks/models/variationally_sparse_gp_minibatch/models.py b/bayesian_benchmarks/models/variationally_sparse_gp_minibatch/models.py index 87dba530..3d89f53f 100644 --- a/bayesian_benchmarks/models/variationally_sparse_gp_minibatch/models.py +++ b/bayesian_benchmarks/models/variationally_sparse_gp_minibatch/models.py @@ -30,8 +30,8 @@ class ARGS: minibatch_size = 1000 initial_likelihood_var = 0.01 self.ARGS = ARGS - self._model = None - self._model_objective = None + self.model = None + self.model_objective = None self._adam_opt = None self._natgrad_opt = None @@ -46,18 +46,18 @@ def fit(self, X, Y): axis=0) # make model if necessary - if self._model is None: + if self.model is None: kernel = gpflow.kernels.SquaredExponential(lengthscale=float(input_dim)**0.5) lik = gpflow.likelihoods.Gaussian(variance=self.ARGS.initial_likelihood_var) - self._model = gpflow.models.SVGP(kernel, likelihood=lik, inducing_variable=Z) + self.model = gpflow.models.SVGP(kernel, likelihood=lik, inducing_variable=Z) @tf.function(autograph=False) def objective(data): - return - self._model.log_marginal_likelihood(data) - self._model_objective = objective + return - self.model.log_marginal_likelihood(data) + self.model_objective = objective - self._model.q_mu.trainable = False - self._model.q_sqrt.trainable = False + self.model.q_mu.trainable = False + self.model.q_sqrt.trainable = False self._natgrad_opt = gpflow.optimizers.NaturalGradient(gamma=self.ARGS.gamma) self._adam_opt = tf.optimizers.Adam(learning_rate=self.ARGS.adam_lr) @@ -67,10 +67,10 @@ def objective(data): iters = self.ARGS.small_iterations # we might have new data - self._model.inducing_variable.Z.assign(Z) - num_outputs = self._model.q_sqrt.shape[0] - self._model.q_mu.assign(np.zeros((self.ARGS.num_inducing, num_outputs))) - self._model.q_sqrt.assign(np.tile(np.eye(self.ARGS.num_inducing)[None], [num_outputs, 1, 1])) + self.model.inducing_variable.Z.assign(Z) + num_outputs = self.model.q_sqrt.shape[0] + self.model.q_mu.assign(np.zeros((self.ARGS.num_inducing, num_outputs))) + self.model.q_sqrt.assign(np.tile(np.eye(self.ARGS.num_inducing)[None], [num_outputs, 1, 1])) batch_size = np.minimum(self.ARGS.minibatch_size, num_data) data = (X, Y) @@ -81,9 +81,9 @@ def objective(data): def objective_closure() -> tf.Tensor: batch = next(data_minibatch_it) - return self._model_objective(batch) + return self.model_objective(batch) - variational_params = [(self._model.q_mu, self._model.q_sqrt)] + variational_params = [(self.model.q_mu, self.model.q_sqrt)] @tf.function def natgrad_step(): @@ -91,14 +91,14 @@ def natgrad_step(): @tf.function def adam_step(): - self._adam_opt.minimize(objective_closure, var_list=self._model.trainable_variables) + self._adam_opt.minimize(objective_closure, var_list=self.model.trainable_variables) for _ in trange(iters): natgrad_step() adam_step() def predict(self, Xs): - return self._model.predict_y(Xs) + return self.model.predict_y(Xs) def sample(self, Xs, num_samples): m, v = self.predict(Xs) @@ -126,8 +126,8 @@ class ARGS: self.ARGS = ARGS self.K = K - self._model = None - self._model_objective = None + self.model = None + self.model_objective = None self._opt = None def fit(self, X, Y): @@ -138,7 +138,7 @@ def fit(self, X, Y): else: Z = X.copy() - if self._model is None: + if self.model is None: if self.K == 2: lik = gpflow.likelihoods.Bernoulli() num_latent = 1 @@ -147,13 +147,13 @@ def fit(self, X, Y): num_latent = self.K kernel = gpflow.kernels.SquaredExponential(lengthscale=float(input_dim)**0.5) - self._model = gpflow.models.SVGP(kernel, likelihood=lik, inducing_variable=Z, + self.model = gpflow.models.SVGP(kernel, likelihood=lik, inducing_variable=Z, whiten=False, num_latent=num_latent) @tf.function(autograph=False) def objective(data): - return - self._model.log_marginal_likelihood(data) - self._model_objective = objective + return - self.model.log_marginal_likelihood(data) + self.model_objective = objective self._opt = tf.optimizers.Adam(self.ARGS.adam_lr) @@ -163,10 +163,10 @@ def objective(data): iters = self.ARGS.small_iterations # we might have new data - self._model.inducing_variable.Z.assign(Z) - num_outputs = self._model.q_sqrt.shape[0] - self._model.q_mu.assign(np.zeros((self.ARGS.num_inducing, num_outputs))) - self._model.q_sqrt.assign(np.tile(np.eye(self.ARGS.num_inducing)[None], [num_outputs, 1, 1])) + self.model.inducing_variable.Z.assign(Z) + num_outputs = self.model.q_sqrt.shape[0] + self.model.q_mu.assign(np.zeros((self.ARGS.num_inducing, num_outputs))) + self.model.q_sqrt.assign(np.tile(np.eye(self.ARGS.num_inducing)[None], [num_outputs, 1, 1])) batch_size = np.minimum(self.ARGS.minibatch_size, num_data) data = (X, Y) @@ -177,17 +177,17 @@ def objective(data): def objective_closure() -> tf.Tensor: batch = next(data_minibatch_it) - return self._model_objective(batch) + return self.model_objective(batch) @tf.function def adam_step(): - self._opt.minimize(objective_closure, var_list=self._model.trainable_variables) + self._opt.minimize(objective_closure, var_list=self.model.trainable_variables) for _ in trange(iters): adam_step() def predict(self, Xs): - m, v = self._model.predict_y(Xs) + m, v = self.model.predict_y(Xs) if self.K == 2: # convert Bernoulli to onehot return np.concatenate([1 - m, m], axis=1) From cd30f1680fd9ee06551af14e8de65637e24fe525 Mon Sep 17 00:00:00 2001 From: ST John Date: Mon, 26 Sep 2022 15:39:11 +0300 Subject: [PATCH 8/8] fix --- bayesian_benchmarks/models/variationally_sparse_gp/models.py | 4 ++-- .../models/variationally_sparse_gp_minibatch/models.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bayesian_benchmarks/models/variationally_sparse_gp/models.py b/bayesian_benchmarks/models/variationally_sparse_gp/models.py index edf13f79..37cd03e1 100644 --- a/bayesian_benchmarks/models/variationally_sparse_gp/models.py +++ b/bayesian_benchmarks/models/variationally_sparse_gp/models.py @@ -36,7 +36,7 @@ def fit(self, X, Y): # make model if necessary if self.model is None: data = (tf.Variable(X, trainable=False), tf.Variable(Y, trainable=False)) - lengthscales = np.full(D, float(D)**0.5) + lengthscales = np.full(input_dim, float(input_dim)**0.5) kernel = gpflow.kernels.SquaredExponential(lengthscales=lengthscales) # Gaussian likelihood: use SGPR self.model = gpflow.models.SGPR(data, kernel, inducing_variable=Z, @@ -99,7 +99,7 @@ def fit(self, X, Y): lik = gpflow.likelihoods.MultiClass(self.K) num_latent_gps = self.K - lengthscales = np.full(D, float(D)**0.5) + lengthscales = np.full(input_dim, float(input_dim)**0.5) kernel = gpflow.kernels.SquaredExponential(lengthscales=lengthscales) self.model = gpflow.models.SVGP(kernel, lik, inducing_variable=Z, diff --git a/bayesian_benchmarks/models/variationally_sparse_gp_minibatch/models.py b/bayesian_benchmarks/models/variationally_sparse_gp_minibatch/models.py index bbb430be..9f05de2d 100644 --- a/bayesian_benchmarks/models/variationally_sparse_gp_minibatch/models.py +++ b/bayesian_benchmarks/models/variationally_sparse_gp_minibatch/models.py @@ -48,7 +48,7 @@ def fit(self, X, Y): # make model if necessary if self.model is None: - lengthscales = np.full(D, float(D)**0.5) + lengthscales = np.full(input_dim, float(input_dim)**0.5) kernel = gpflow.kernels.SquaredExponential(lengthscales=lengthscales) lik = gpflow.likelihoods.Gaussian(variance=self.ARGS.initial_likelihood_var) self.model = gpflow.models.SVGP(kernel, likelihood=lik, inducing_variable=Z, num_data=num_data) @@ -140,7 +140,7 @@ def fit(self, X, Y): lik = gpflow.likelihoods.MultiClass(self.K) num_latent_gps = self.K - lengthscales = np.full(D, float(D)**0.5) + lengthscales = np.full(input_dim, float(input_dim)**0.5) kernel = gpflow.kernels.SquaredExponential(lengthscales=lengthscales) self.model = gpflow.models.SVGP(kernel, likelihood=lik, inducing_variable=Z, num_latent_gps=num_latent_gps)