From dd475bb73d5cbf9dd97c22edfd6d8e529d91c8c1 Mon Sep 17 00:00:00 2001 From: Yohann Lereclus Date: Fri, 6 Mar 2020 11:05:17 +0100 Subject: [PATCH 01/11] add package test functions --- test_functions/__init__.py | 3 ++ test_functions/rastrigin.py | 40 ++++++++++++++++++++++++++ test_functions/rosenbrock.py | 31 ++++++++++++++++++++ test_functions/test_function.py | 50 +++++++++++++++++++++++++++++++++ 4 files changed, 124 insertions(+) create mode 100644 test_functions/__init__.py create mode 100644 test_functions/rastrigin.py create mode 100644 test_functions/rosenbrock.py create mode 100644 test_functions/test_function.py diff --git a/test_functions/__init__.py b/test_functions/__init__.py new file mode 100644 index 0000000..85a1c7c --- /dev/null +++ b/test_functions/__init__.py @@ -0,0 +1,3 @@ +from .rosenbrock import Rosenbrock +from .rastrigin import Rastrigin + diff --git a/test_functions/rastrigin.py b/test_functions/rastrigin.py new file mode 100644 index 0000000..b215a2a --- /dev/null +++ b/test_functions/rastrigin.py @@ -0,0 +1,40 @@ +import torch +import math + +from .test_function import TestFunction + + +class Rastrigin(TestFunction): + r"""Rastrigin test function. + + Example: + >>> import test_functions + >>> rastrigin = test_functions.Rastrigin() + >>> x = torch.linspace(rastrigin.x_domain[0], rastrigin.x_domain[1], rastrigin.num_pt) + >>> y = torch.linspace(rastrigin.y_domain[0], rastrigin.y_domain[1], rastrigin.num_pt) + >>> Y, X = torch.meshgrid(x, y) + >>> Z = rastrigin([X, Y]) + + __ https://en.wikipedia.org/wiki/Test_functions_for_optimization + __ https://en.wikipedia.org/wiki/Rastrigin_function + """ + def __init__(self): + super(Rastrigin, self).__init__( + x_domain=(-4.5, 4.5), + y_domain=(-4.5, 4.5), + minimum=(0, 0), + initial_state=(-2.0, 3.5), + levels=20 + ) + + def __call__(self, tensor, lib=torch): + x, y = tensor + A = 10 + f = ( + A * 2 + + (x ** 2 - A * lib.cos(x * math.pi * 2)) + + (y ** 2 - A * lib.cos(y * math.pi * 2)) + ) + return f + + diff --git a/test_functions/rosenbrock.py b/test_functions/rosenbrock.py new file mode 100644 index 0000000..835a105 --- /dev/null +++ b/test_functions/rosenbrock.py @@ -0,0 +1,31 @@ +import torch + +from .test_function import TestFunction + + +class Rosenbrock(TestFunction): + r"""Rosenbrock test function. + + Example: + >>> import test_functions + >>> rosenbrock = test_functions.Rosenbrock() + >>> x = torch.linspace(rosenbrock.x_domain[0], rosenbrock.x_domain[1], rosenbrock.num_pt) + >>> y = torch.linspace(rosenbrock.y_domain[0], rosenbrock.y_domain[1], rosenbrock.num_pt) + >>> Y, X = torch.meshgrid(x, y) + >>> Z = rosenbrock([X, Y]) + + __ https://en.wikipedia.org/wiki/Test_functions_for_optimization + __ https://en.wikipedia.org/wiki/Rosenbrock_function + """ + def __init__(self): + super(Rosenbrock, self).__init__( + x_domain=(-2.0, 2.0), + y_domain=(-1.0, 3.0), + minimum=(1.0, 1.0), + initial_state=(-2.0, 2.0), + levels=90, + ) + + def __call__(self, tensor, lib=torch): + x, y = tensor + return (1 - x) ** 2 + 100 * (y - x ** 2) ** 2 diff --git a/test_functions/test_function.py b/test_functions/test_function.py new file mode 100644 index 0000000..2b9a872 --- /dev/null +++ b/test_functions/test_function.py @@ -0,0 +1,50 @@ +class TestFunction: + r"""Super class to implement a test function. + + Arguments: + x_domain: iterable of int for the x search domain + y_domain: iterable of int for the y search domain + minimum: global minimum (x0, y0) of the + function for the given domain + initial_state: starting point + num_pt: number of point in the search domain + levels: determines the number and positions of the contour lines / regions. + """ + def __init__(self, x_domain: iter, y_domain: iter, minimum: iter, initial_state: iter, + num_pt: int = 250, levels: int = 50): + self._x_domain = x_domain + self._y_domain = y_domain + self._minimum = minimum + self._initial_state = initial_state + self._num_pt = num_pt + self._levels = levels + + def __call__(self, tensor, lib): + raise NotImplementedError + + def __name__(self): + return self.__class__.__name__ + + @property + def x_domain(self): + return self._x_domain + + @property + def y_domain(self): + return self._y_domain + + @property + def num_pt(self): + return self._num_pt + + @property + def minimum(self): + return self._minimum + + @property + def initial_state(self): + return self._initial_state + + @property + def levels(self): + return self._levels From 73837f7decde33277fdac556602564300b28e7ce Mon Sep 17 00:00:00 2001 From: Yohann Lereclus Date: Fri, 6 Mar 2020 11:05:52 +0100 Subject: [PATCH 02/11] refactor viz_optimizers to use test_functions package --- examples/viz_optimizers.py | 198 ++++++++++++++++++++++--------------- 1 file changed, 117 insertions(+), 81 deletions(-) diff --git a/examples/viz_optimizers.py b/examples/viz_optimizers.py index e4f7472..0ad7e52 100644 --- a/examples/viz_optimizers.py +++ b/examples/viz_optimizers.py @@ -4,25 +4,18 @@ import torch from hyperopt import fmin, tpe, hp import matplotlib.pyplot as plt - +from test_functions import * plt.style.use('seaborn-white') -def rosenbrock(tensor): - # https://en.wikipedia.org/wiki/Test_functions_for_optimization - x, y = tensor - return (1 - x) ** 2 + 100 * (y - x ** 2) ** 2 - - -def rastrigin(tensor, lib=torch): +def ackley(tensor, lib=torch): # https://en.wikipedia.org/wiki/Test_functions_for_optimization x, y = tensor - A = 10 f = ( - A * 2 - + (x ** 2 - A * lib.cos(x * math.pi * 2)) - + (y ** 2 - A * lib.cos(y * math.pi * 2)) + -20 * lib.exp(-0.2 * lib.sqrt(0.5 * (x * x + y * y))) + - lib.exp(0.5 * lib.cos(2 * math.pi * x) + lib.cos(2 * math.pi * y)) + + math.e + 20 ) return f @@ -45,83 +38,135 @@ def execute_steps( return steps -def objective_rastrigin(params): - lr = params['lr'] - optimizer_class = params['optimizer_class'] - initial_state = (-2.0, 3.5) - minimum = (0, 0) - optimizer_config = dict(lr=lr) - num_iter = 100 - steps = execute_steps( - rastrigin, initial_state, optimizer_class, optimizer_config, num_iter - ) - return (steps[0][-1] - minimum[0]) ** 2 + (steps[1][-1] - minimum[1]) ** 2 - - -def objective_rosenbrok(params): +# def objective_rastrigin(params): +# lr = params['lr'] +# optimizer_class = params['optimizer_class'] +# initial_state = (-2.0, 3.5) +# minimum = (0, 0) +# optimizer_config = dict(lr=lr) +# num_iter = 100 +# steps = execute_steps( +# rastrigin, initial_state, optimizer_class, optimizer_config, num_iter +# ) +# return (steps[0][-1] - minimum[0]) ** 2 + (steps[1][-1] - minimum[1]) ** 2 +# +# +# def objective_rosenbrok(params): +# lr = params['lr'] +# optimizer_class = params['optimizer_class'] +# minimum = (1.0, 1.0) +# initial_state = (-2.0, 2.0) +# optimizer_config = dict(lr=lr) +# num_iter = 100 +# steps = execute_steps( +# rosenbrock, initial_state, optimizer_class, optimizer_config, num_iter +# ) +# return (steps[0][-1] - minimum[0]) ** 2 + (steps[1][-1] - minimum[1]) ** 2 +# +# +# def objective_ackley(params): +# lr = params['lr'] +# optimizer_class = params['optimizer_class'] +# minimum = (0, 0) +# initial_state = (-2.0, 4.0) +# optimizer_config = dict(lr=lr) +# num_iter = 100 +# steps = execute_steps( +# ackley, initial_state, optimizer_class, optimizer_config, num_iter +# ) +# return (steps[0][-1] - minimum[0]) ** 2 + (steps[1][-1] - minimum[1]) ** 2 + + +def objective(params, func): lr = params['lr'] optimizer_class = params['optimizer_class'] - minimum = (1.0, 1.0) - initial_state = (-2.0, 2.0) + minimum = func.minimum + initial_state = func.initial_state optimizer_config = dict(lr=lr) num_iter = 100 steps = execute_steps( - rosenbrock, initial_state, optimizer_class, optimizer_config, num_iter + ackley, initial_state, optimizer_class, optimizer_config, num_iter ) return (steps[0][-1] - minimum[0]) ** 2 + (steps[1][-1] - minimum[1]) ** 2 -def plot_rastrigin(grad_iter, optimizer_name, lr): - x = np.linspace(-4.5, 4.5, 250) - y = np.linspace(-4.5, 4.5, 250) - minimum = (0, 0) - - X, Y = np.meshgrid(x, y) - Z = rastrigin([X, Y], lib=np) +# def plot_rastrigin(grad_iter, optimizer_name, lr): +# x = np.linspace(-4.5, 4.5, 250) +# y = np.linspace(-4.5, 4.5, 250) +# minimum = (0, 0) +# +# X, Y = np.meshgrid(x, y) +# Z = rastrigin([X, Y], lib=np) +# +# iter_x, iter_y = grad_iter[0, :], grad_iter[1, :] +# +# fig = plt.figure(figsize=(8, 8)) +# +# ax = fig.add_subplot(1, 1, 1) +# ax.contour(X, Y, Z, 20, cmap='jet') +# ax.plot(iter_x, iter_y, color='r', marker='x') +# ax.set_title( +# f'Rastrigin func: {optimizer_name} with ' +# f'{len(iter_x)} iterations, lr={lr:.6}' +# ) +# plt.plot(*minimum, 'gD') +# plt.plot(iter_x[-1], iter_y[-1], 'rD') +# plt.savefig(f'docs/rastrigin_{optimizer_name}.png') +# +# +# def plot_rosenbrok(grad_iter, optimizer_name, lr): +# x = np.linspace(-2, 2, 250) +# y = np.linspace(-1, 3, 250) +# minimum = (1.0, 1.0) +# +# X, Y = np.meshgrid(x, y) +# Z = rosenbrock([X, Y]) +# +# iter_x, iter_y = grad_iter[0, :], grad_iter[1, :] +# +# fig = plt.figure(figsize=(8, 8)) +# +# ax = fig.add_subplot(1, 1, 1) +# ax.contour(X, Y, Z, 90, cmap='jet') +# ax.plot(iter_x, iter_y, color='r', marker='x') +# +# ax.set_title( +# f'Rosenbrock func: {optimizer_name} with {len(iter_x)} ' +# f'iterations, lr={lr:.6}' +# ) +# plt.plot(*minimum, 'gD') +# plt.plot(iter_x[-1], iter_y[-1], 'rD') +# plt.savefig(f'docs/rosenbrock_{optimizer_name}.png') + + +def plot(func, grad_iter, optimizer_name, lr): + x = torch.linspace(func.x_domain[0], func.x_domain[1], func.num_pt) + y = torch.linspace(func.y_domain[0], func.y_domain[1], func.num_pt) + minimum = func.minimum + + Y, X = torch.meshgrid(x, y) + Z = func([X, Y]) iter_x, iter_y = grad_iter[0, :], grad_iter[1, :] fig = plt.figure(figsize=(8, 8)) ax = fig.add_subplot(1, 1, 1) - ax.contour(X, Y, Z, 20, cmap='jet') - ax.plot(iter_x, iter_y, color='r', marker='x') - ax.set_title( - f'Rastrigin func: {optimizer_name} with ' - f'{len(iter_x)} iterations, lr={lr:.6}' - ) - plt.plot(*minimum, 'gD') - plt.plot(iter_x[-1], iter_y[-1], 'rD') - plt.savefig(f'docs/rastrigin_{optimizer_name}.png') - - -def plot_rosenbrok(grad_iter, optimizer_name, lr): - x = np.linspace(-2, 2, 250) - y = np.linspace(-1, 3, 250) - minimum = (1.0, 1.0) - - X, Y = np.meshgrid(x, y) - Z = rosenbrock([X, Y]) - - iter_x, iter_y = grad_iter[0, :], grad_iter[1, :] - - fig = plt.figure(figsize=(8, 8)) - - ax = fig.add_subplot(1, 1, 1) - ax.contour(X, Y, Z, 90, cmap='jet') + ax.contour(X, Y, Z, func.levels, cmap='jet') ax.plot(iter_x, iter_y, color='r', marker='x') + func_name = func.__name__() ax.set_title( - f'Rosenbrock func: {optimizer_name} with {len(iter_x)} ' + f'{func_name} func: {optimizer_name} with {len(iter_x)} ' f'iterations, lr={lr:.6}' ) plt.plot(*minimum, 'gD') plt.plot(iter_x[-1], iter_y[-1], 'rD') - plt.savefig(f'docs/rosenbrock_{optimizer_name}.png') + plt.savefig(f'docs/{func_name}_{optimizer_name}.png') def execute_experiments( - optimizers, objective, func, plot_func, initial_state, seed=1 + optimizers, func, seed=1 ): seed = seed for item in optimizers: @@ -131,7 +176,7 @@ def execute_experiments( 'lr': hp.loguniform('lr', lr_low, lr_hi), } best = fmin( - fn=objective, + fn=lambda x: objective(x, func), space=space, algo=tpe.suggest, max_evals=200, @@ -141,12 +186,12 @@ def execute_experiments( steps = execute_steps( func, - initial_state, + func.initial_state, optimizer_class, {'lr': best['lr']}, num_iter=500, ) - plot_func(steps, optimizer_class.__name__, best['lr']) + plot(func, steps, optimizer_class.__name__, best['lr']) if __name__ == '__main__': @@ -168,18 +213,9 @@ def execute_experiments( (optim.SGDW, -8, -1.5), (optim.PID, -8, -1.0), ] - execute_experiments( - optimizers, - objective_rastrigin, - rastrigin, - plot_rastrigin, - (-2.0, 3.5), - ) - execute_experiments( - optimizers, - objective_rosenbrok, - rosenbrock, - plot_rosenbrok, - (-2.0, 2.0), - ) + for test_func in [Rastrigin(), Rosenbrock()]: + execute_experiments( + optimizers, + test_func, + ) \ No newline at end of file From bcc38f1ae18bf69c30ee3e207997d0b3a39c1350 Mon Sep 17 00:00:00 2001 From: Yohann Lereclus Date: Fri, 6 Mar 2020 11:11:15 +0100 Subject: [PATCH 03/11] remove commented func --- examples/viz_optimizers.py | 115 +++---------------------------------- 1 file changed, 8 insertions(+), 107 deletions(-) diff --git a/examples/viz_optimizers.py b/examples/viz_optimizers.py index 0ad7e52..4e8732d 100644 --- a/examples/viz_optimizers.py +++ b/examples/viz_optimizers.py @@ -1,27 +1,16 @@ -import math +import matplotlib.pyplot as plt import numpy as np -import torch_optimizer as optim import torch from hyperopt import fmin, tpe, hp -import matplotlib.pyplot as plt + +import torch_optimizer as optim from test_functions import * plt.style.use('seaborn-white') -def ackley(tensor, lib=torch): - # https://en.wikipedia.org/wiki/Test_functions_for_optimization - x, y = tensor - f = ( - -20 * lib.exp(-0.2 * lib.sqrt(0.5 * (x * x + y * y))) - - lib.exp(0.5 * lib.cos(2 * math.pi * x) + lib.cos(2 * math.pi * y)) - + math.e + 20 - ) - return f - - def execute_steps( - func, initial_state, optimizer_class, optimizer_config, num_iter=500 + func, initial_state, optimizer_class, optimizer_config, num_iter=500 ): x = torch.Tensor(initial_state).requires_grad_(True) optimizer = optimizer_class([x], **optimizer_config) @@ -38,45 +27,6 @@ def execute_steps( return steps -# def objective_rastrigin(params): -# lr = params['lr'] -# optimizer_class = params['optimizer_class'] -# initial_state = (-2.0, 3.5) -# minimum = (0, 0) -# optimizer_config = dict(lr=lr) -# num_iter = 100 -# steps = execute_steps( -# rastrigin, initial_state, optimizer_class, optimizer_config, num_iter -# ) -# return (steps[0][-1] - minimum[0]) ** 2 + (steps[1][-1] - minimum[1]) ** 2 -# -# -# def objective_rosenbrok(params): -# lr = params['lr'] -# optimizer_class = params['optimizer_class'] -# minimum = (1.0, 1.0) -# initial_state = (-2.0, 2.0) -# optimizer_config = dict(lr=lr) -# num_iter = 100 -# steps = execute_steps( -# rosenbrock, initial_state, optimizer_class, optimizer_config, num_iter -# ) -# return (steps[0][-1] - minimum[0]) ** 2 + (steps[1][-1] - minimum[1]) ** 2 -# -# -# def objective_ackley(params): -# lr = params['lr'] -# optimizer_class = params['optimizer_class'] -# minimum = (0, 0) -# initial_state = (-2.0, 4.0) -# optimizer_config = dict(lr=lr) -# num_iter = 100 -# steps = execute_steps( -# ackley, initial_state, optimizer_class, optimizer_config, num_iter -# ) -# return (steps[0][-1] - minimum[0]) ** 2 + (steps[1][-1] - minimum[1]) ** 2 - - def objective(params, func): lr = params['lr'] optimizer_class = params['optimizer_class'] @@ -85,66 +35,17 @@ def objective(params, func): optimizer_config = dict(lr=lr) num_iter = 100 steps = execute_steps( - ackley, initial_state, optimizer_class, optimizer_config, num_iter + func, initial_state, optimizer_class, optimizer_config, num_iter ) return (steps[0][-1] - minimum[0]) ** 2 + (steps[1][-1] - minimum[1]) ** 2 -# def plot_rastrigin(grad_iter, optimizer_name, lr): -# x = np.linspace(-4.5, 4.5, 250) -# y = np.linspace(-4.5, 4.5, 250) -# minimum = (0, 0) -# -# X, Y = np.meshgrid(x, y) -# Z = rastrigin([X, Y], lib=np) -# -# iter_x, iter_y = grad_iter[0, :], grad_iter[1, :] -# -# fig = plt.figure(figsize=(8, 8)) -# -# ax = fig.add_subplot(1, 1, 1) -# ax.contour(X, Y, Z, 20, cmap='jet') -# ax.plot(iter_x, iter_y, color='r', marker='x') -# ax.set_title( -# f'Rastrigin func: {optimizer_name} with ' -# f'{len(iter_x)} iterations, lr={lr:.6}' -# ) -# plt.plot(*minimum, 'gD') -# plt.plot(iter_x[-1], iter_y[-1], 'rD') -# plt.savefig(f'docs/rastrigin_{optimizer_name}.png') -# -# -# def plot_rosenbrok(grad_iter, optimizer_name, lr): -# x = np.linspace(-2, 2, 250) -# y = np.linspace(-1, 3, 250) -# minimum = (1.0, 1.0) -# -# X, Y = np.meshgrid(x, y) -# Z = rosenbrock([X, Y]) -# -# iter_x, iter_y = grad_iter[0, :], grad_iter[1, :] -# -# fig = plt.figure(figsize=(8, 8)) -# -# ax = fig.add_subplot(1, 1, 1) -# ax.contour(X, Y, Z, 90, cmap='jet') -# ax.plot(iter_x, iter_y, color='r', marker='x') -# -# ax.set_title( -# f'Rosenbrock func: {optimizer_name} with {len(iter_x)} ' -# f'iterations, lr={lr:.6}' -# ) -# plt.plot(*minimum, 'gD') -# plt.plot(iter_x[-1], iter_y[-1], 'rD') -# plt.savefig(f'docs/rosenbrock_{optimizer_name}.png') - - def plot(func, grad_iter, optimizer_name, lr): x = torch.linspace(func.x_domain[0], func.x_domain[1], func.num_pt) y = torch.linspace(func.y_domain[0], func.y_domain[1], func.num_pt) minimum = func.minimum - Y, X = torch.meshgrid(x, y) + X, Y = torch.meshgrid(x, y) Z = func([X, Y]) iter_x, iter_y = grad_iter[0, :], grad_iter[1, :] @@ -166,7 +67,7 @@ def plot(func, grad_iter, optimizer_name, lr): def execute_experiments( - optimizers, func, seed=1 + optimizers, func, seed=1 ): seed = seed for item in optimizers: @@ -218,4 +119,4 @@ def execute_experiments( execute_experiments( optimizers, test_func, - ) \ No newline at end of file + ) From dfe4c736add8cb80f0cbb1af2e9917bd40b2befb Mon Sep 17 00:00:00 2001 From: Yohann Lereclus Date: Fri, 6 Mar 2020 11:16:30 +0100 Subject: [PATCH 04/11] add print test func --- examples/viz_optimizers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/viz_optimizers.py b/examples/viz_optimizers.py index 4e8732d..cc034d0 100644 --- a/examples/viz_optimizers.py +++ b/examples/viz_optimizers.py @@ -115,7 +115,8 @@ def execute_experiments( (optim.PID, -8, -1.0), ] - for test_func in [Rastrigin(), Rosenbrock()]: + for test_func in [Rosenbrock()]: + print(f"Test function {test_func.__name__()}") execute_experiments( optimizers, test_func, From 021eb354326875b8273017758881170637dcbfef Mon Sep 17 00:00:00 2001 From: Yohann Lereclus Date: Fri, 6 Mar 2020 11:36:14 +0100 Subject: [PATCH 05/11] fix docstring too long --- test_functions/rastrigin.py | 12 ++++++++++-- test_functions/rosenbrock.py | 12 ++++++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/test_functions/rastrigin.py b/test_functions/rastrigin.py index b215a2a..a065cd4 100644 --- a/test_functions/rastrigin.py +++ b/test_functions/rastrigin.py @@ -10,8 +10,16 @@ class Rastrigin(TestFunction): Example: >>> import test_functions >>> rastrigin = test_functions.Rastrigin() - >>> x = torch.linspace(rastrigin.x_domain[0], rastrigin.x_domain[1], rastrigin.num_pt) - >>> y = torch.linspace(rastrigin.y_domain[0], rastrigin.y_domain[1], rastrigin.num_pt) + >>> x = torch.linspace( + rastrigin.x_domain[0], + rastrigin.x_domain[1], + rastrigin.num_pt + ) + >>> y = torch.linspace( + rastrigin.y_domain[0], + rastrigin.y_domain[1], + rastrigin.num_pt + ) >>> Y, X = torch.meshgrid(x, y) >>> Z = rastrigin([X, Y]) diff --git a/test_functions/rosenbrock.py b/test_functions/rosenbrock.py index 835a105..c42fa8f 100644 --- a/test_functions/rosenbrock.py +++ b/test_functions/rosenbrock.py @@ -9,8 +9,16 @@ class Rosenbrock(TestFunction): Example: >>> import test_functions >>> rosenbrock = test_functions.Rosenbrock() - >>> x = torch.linspace(rosenbrock.x_domain[0], rosenbrock.x_domain[1], rosenbrock.num_pt) - >>> y = torch.linspace(rosenbrock.y_domain[0], rosenbrock.y_domain[1], rosenbrock.num_pt) + >>> x = torch.linspace( + rosenbrock.x_domain[0], + rosenbrock.x_domain[1], + rosenbrock.num_pt + ) + >>> y = torch.linspace( + rosenbrock.y_domain[0], + rosenbrock.y_domain[1], + rosenbrock.num_pt + ) >>> Y, X = torch.meshgrid(x, y) >>> Z = rosenbrock([X, Y]) From 62282fac1eae257ebc067b84479bc36a27d93532 Mon Sep 17 00:00:00 2001 From: Yohann Lereclus Date: Fri, 6 Mar 2020 11:36:42 +0100 Subject: [PATCH 06/11] fix blank line at the end of file --- test_functions/rastrigin.py | 4 +--- test_functions/rosenbrock.py | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/test_functions/rastrigin.py b/test_functions/rastrigin.py index a065cd4..c0d6ab1 100644 --- a/test_functions/rastrigin.py +++ b/test_functions/rastrigin.py @@ -43,6 +43,4 @@ def __call__(self, tensor, lib=torch): + (x ** 2 - A * lib.cos(x * math.pi * 2)) + (y ** 2 - A * lib.cos(y * math.pi * 2)) ) - return f - - + return f \ No newline at end of file diff --git a/test_functions/rosenbrock.py b/test_functions/rosenbrock.py index c42fa8f..913da2c 100644 --- a/test_functions/rosenbrock.py +++ b/test_functions/rosenbrock.py @@ -10,7 +10,7 @@ class Rosenbrock(TestFunction): >>> import test_functions >>> rosenbrock = test_functions.Rosenbrock() >>> x = torch.linspace( - rosenbrock.x_domain[0], + rosenbrock.x_domain[0], rosenbrock.x_domain[1], rosenbrock.num_pt ) @@ -36,4 +36,4 @@ def __init__(self): def __call__(self, tensor, lib=torch): x, y = tensor - return (1 - x) ** 2 + 100 * (y - x ** 2) ** 2 + return (1 - x) ** 2 + 100 * (y - x ** 2) ** 2 \ No newline at end of file From b5079b000da911099d0dd71ac373a396bd297373 Mon Sep 17 00:00:00 2001 From: Yohann Lereclus Date: Fri, 6 Mar 2020 11:37:51 +0100 Subject: [PATCH 07/11] fix import --- examples/viz_optimizers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/viz_optimizers.py b/examples/viz_optimizers.py index cc034d0..ef65dad 100644 --- a/examples/viz_optimizers.py +++ b/examples/viz_optimizers.py @@ -4,7 +4,7 @@ from hyperopt import fmin, tpe, hp import torch_optimizer as optim -from test_functions import * +from test_functions import Rastrigin, Rosenbrock plt.style.use('seaborn-white') @@ -115,7 +115,7 @@ def execute_experiments( (optim.PID, -8, -1.0), ] - for test_func in [Rosenbrock()]: + for test_func in [Rastrigin(), Rosenbrock()]: print(f"Test function {test_func.__name__()}") execute_experiments( optimizers, From 1947e6a03e1b3636df7d739d73db05d215523647 Mon Sep 17 00:00:00 2001 From: Yohann Lereclus Date: Fri, 6 Mar 2020 11:41:26 +0100 Subject: [PATCH 08/11] black format --- test_functions/rastrigin.py | 11 ++++++----- test_functions/rosenbrock.py | 3 ++- test_functions/test_function.py | 12 ++++++++++-- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/test_functions/rastrigin.py b/test_functions/rastrigin.py index c0d6ab1..19cac36 100644 --- a/test_functions/rastrigin.py +++ b/test_functions/rastrigin.py @@ -26,21 +26,22 @@ class Rastrigin(TestFunction): __ https://en.wikipedia.org/wiki/Test_functions_for_optimization __ https://en.wikipedia.org/wiki/Rastrigin_function """ + def __init__(self): super(Rastrigin, self).__init__( x_domain=(-4.5, 4.5), y_domain=(-4.5, 4.5), minimum=(0, 0), initial_state=(-2.0, 3.5), - levels=20 + levels=20, ) def __call__(self, tensor, lib=torch): x, y = tensor A = 10 f = ( - A * 2 - + (x ** 2 - A * lib.cos(x * math.pi * 2)) - + (y ** 2 - A * lib.cos(y * math.pi * 2)) + A * 2 + + (x ** 2 - A * lib.cos(x * math.pi * 2)) + + (y ** 2 - A * lib.cos(y * math.pi * 2)) ) - return f \ No newline at end of file + return f diff --git a/test_functions/rosenbrock.py b/test_functions/rosenbrock.py index 913da2c..2792b10 100644 --- a/test_functions/rosenbrock.py +++ b/test_functions/rosenbrock.py @@ -25,6 +25,7 @@ class Rosenbrock(TestFunction): __ https://en.wikipedia.org/wiki/Test_functions_for_optimization __ https://en.wikipedia.org/wiki/Rosenbrock_function """ + def __init__(self): super(Rosenbrock, self).__init__( x_domain=(-2.0, 2.0), @@ -36,4 +37,4 @@ def __init__(self): def __call__(self, tensor, lib=torch): x, y = tensor - return (1 - x) ** 2 + 100 * (y - x ** 2) ** 2 \ No newline at end of file + return (1 - x) ** 2 + 100 * (y - x ** 2) ** 2 diff --git a/test_functions/test_function.py b/test_functions/test_function.py index 2b9a872..cc93f50 100644 --- a/test_functions/test_function.py +++ b/test_functions/test_function.py @@ -10,8 +10,16 @@ class TestFunction: num_pt: number of point in the search domain levels: determines the number and positions of the contour lines / regions. """ - def __init__(self, x_domain: iter, y_domain: iter, minimum: iter, initial_state: iter, - num_pt: int = 250, levels: int = 50): + + def __init__( + self, + x_domain: iter, + y_domain: iter, + minimum: iter, + initial_state: iter, + num_pt: int = 250, + levels: int = 50, + ): self._x_domain = x_domain self._y_domain = y_domain self._minimum = minimum From 8955f1beb2ce4883b5d28cf3131b94faabcb4ea9 Mon Sep 17 00:00:00 2001 From: Yohann Lereclus Date: Fri, 6 Mar 2020 11:44:50 +0100 Subject: [PATCH 09/11] fix end line and varaible outer scope --- examples/viz_optimizers.py | 2 +- test_functions/__init__.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/examples/viz_optimizers.py b/examples/viz_optimizers.py index ef65dad..5654751 100644 --- a/examples/viz_optimizers.py +++ b/examples/viz_optimizers.py @@ -100,7 +100,7 @@ def execute_experiments( # Each optimizer has tweaked search space to produce better plots and # help to converge on better lr faster. - optimizers = [ + list_optimizers = [ # Adam based (optim.AdaBound, -8, 0.3), (optim.AdaMod, -8, 0.2), diff --git a/test_functions/__init__.py b/test_functions/__init__.py index 85a1c7c..b3d692d 100644 --- a/test_functions/__init__.py +++ b/test_functions/__init__.py @@ -1,3 +1,2 @@ from .rosenbrock import Rosenbrock -from .rastrigin import Rastrigin - +from .rastrigin import Rastrigin \ No newline at end of file From aac66a8fd9fd930001c72841ff5f974a68d2b9bc Mon Sep 17 00:00:00 2001 From: Yohann Lereclus Date: Fri, 6 Mar 2020 11:48:16 +0100 Subject: [PATCH 10/11] add base docstring --- examples/viz_optimizers.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/examples/viz_optimizers.py b/examples/viz_optimizers.py index 5654751..6ff3531 100644 --- a/examples/viz_optimizers.py +++ b/examples/viz_optimizers.py @@ -12,6 +12,9 @@ def execute_steps( func, initial_state, optimizer_class, optimizer_config, num_iter=500 ): + """ + Execute one steps of the optimizer + """ x = torch.Tensor(initial_state).requires_grad_(True) optimizer = optimizer_class([x], **optimizer_config) steps = [] @@ -28,6 +31,9 @@ def execute_steps( def objective(params, func): + """ + Execute objective + """ lr = params['lr'] optimizer_class = params['optimizer_class'] minimum = func.minimum @@ -41,6 +47,9 @@ def objective(params, func): def plot(func, grad_iter, optimizer_name, lr): + """ + Plot result of a simulation + """ x = torch.linspace(func.x_domain[0], func.x_domain[1], func.num_pt) y = torch.linspace(func.y_domain[0], func.y_domain[1], func.num_pt) minimum = func.minimum @@ -69,6 +78,9 @@ def plot(func, grad_iter, optimizer_name, lr): def execute_experiments( optimizers, func, seed=1 ): + """ + Execute simulation on a list of optimizers using a test function + """ seed = seed for item in optimizers: optimizer_class, lr_low, lr_hi = item From b631c1d232353fc0f73188a8487b389160850ba4 Mon Sep 17 00:00:00 2001 From: Yohann Lereclus Date: Fri, 6 Mar 2020 11:50:42 +0100 Subject: [PATCH 11/11] black format --- examples/viz_optimizers.py | 63 ++++++++++++++------------------------ test_functions/__init__.py | 2 +- 2 files changed, 24 insertions(+), 41 deletions(-) diff --git a/examples/viz_optimizers.py b/examples/viz_optimizers.py index 6ff3531..5443ab3 100644 --- a/examples/viz_optimizers.py +++ b/examples/viz_optimizers.py @@ -6,15 +6,11 @@ import torch_optimizer as optim from test_functions import Rastrigin, Rosenbrock -plt.style.use('seaborn-white') +plt.style.use("seaborn-white") -def execute_steps( - func, initial_state, optimizer_class, optimizer_config, num_iter=500 -): - """ - Execute one steps of the optimizer - """ +def execute_steps(func, initial_state, optimizer_class, optimizer_config, num_iter=500): + """ Execute one steps of the optimizer """ x = torch.Tensor(initial_state).requires_grad_(True) optimizer = optimizer_class([x], **optimizer_config) steps = [] @@ -31,11 +27,9 @@ def execute_steps( def objective(params, func): - """ - Execute objective - """ - lr = params['lr'] - optimizer_class = params['optimizer_class'] + """ Execute objective """ + lr = params["lr"] + optimizer_class = params["optimizer_class"] minimum = func.minimum initial_state = func.initial_state optimizer_config = dict(lr=lr) @@ -47,9 +41,7 @@ def objective(params, func): def plot(func, grad_iter, optimizer_name, lr): - """ - Plot result of a simulation - """ + """ Plot result of a simulation """ x = torch.linspace(func.x_domain[0], func.x_domain[1], func.num_pt) y = torch.linspace(func.y_domain[0], func.y_domain[1], func.num_pt) minimum = func.minimum @@ -62,31 +54,27 @@ def plot(func, grad_iter, optimizer_name, lr): fig = plt.figure(figsize=(8, 8)) ax = fig.add_subplot(1, 1, 1) - ax.contour(X, Y, Z, func.levels, cmap='jet') - ax.plot(iter_x, iter_y, color='r', marker='x') + ax.contour(X, Y, Z, func.levels, cmap="jet") + ax.plot(iter_x, iter_y, color="r", marker="x") func_name = func.__name__() ax.set_title( - f'{func_name} func: {optimizer_name} with {len(iter_x)} ' - f'iterations, lr={lr:.6}' + f"{func_name} func: {optimizer_name} with {len(iter_x)} " + f"iterations, lr={lr:.6}" ) - plt.plot(*minimum, 'gD') - plt.plot(iter_x[-1], iter_y[-1], 'rD') - plt.savefig(f'docs/{func_name}_{optimizer_name}.png') + plt.plot(*minimum, "gD") + plt.plot(iter_x[-1], iter_y[-1], "rD") + plt.savefig(f"docs/{func_name}_{optimizer_name}.png") -def execute_experiments( - optimizers, func, seed=1 -): - """ - Execute simulation on a list of optimizers using a test function - """ +def execute_experiments(optimizers, func, seed=1): + """ Execute simulation on a list of optimizers using a test function """ seed = seed for item in optimizers: optimizer_class, lr_low, lr_hi = item space = { - 'optimizer_class': hp.choice('optimizer_class', [optimizer_class]), - 'lr': hp.loguniform('lr', lr_low, lr_hi), + "optimizer_class": hp.choice("optimizer_class", [optimizer_class]), + "lr": hp.loguniform("lr", lr_low, lr_hi), } best = fmin( fn=lambda x: objective(x, func), @@ -95,19 +83,15 @@ def execute_experiments( max_evals=200, rstate=np.random.RandomState(seed), ) - print(best['lr'], optimizer_class) + print(best["lr"], optimizer_class) steps = execute_steps( - func, - func.initial_state, - optimizer_class, - {'lr': best['lr']}, - num_iter=500, + func, func.initial_state, optimizer_class, {"lr": best["lr"]}, num_iter=500, ) - plot(func, steps, optimizer_class.__name__, best['lr']) + plot(func, steps, optimizer_class.__name__, best["lr"]) -if __name__ == '__main__': +if __name__ == "__main__": # python examples/viz_optimizers.py # Each optimizer has tweaked search space to produce better plots and @@ -130,6 +114,5 @@ def execute_experiments( for test_func in [Rastrigin(), Rosenbrock()]: print(f"Test function {test_func.__name__()}") execute_experiments( - optimizers, - test_func, + list_optimizers, test_func, ) diff --git a/test_functions/__init__.py b/test_functions/__init__.py index b3d692d..9857ea5 100644 --- a/test_functions/__init__.py +++ b/test_functions/__init__.py @@ -1,2 +1,2 @@ from .rosenbrock import Rosenbrock -from .rastrigin import Rastrigin \ No newline at end of file +from .rastrigin import Rastrigin