From 46fa5a5a985f9c624ab15a3dc554b4bfd16cdb1d Mon Sep 17 00:00:00 2001 From: Elizabeth Santorella Date: Thu, 31 Oct 2024 06:30:06 -0700 Subject: [PATCH] Reorganize runner tests following previous class renaming (#2991) Summary: Pull Request resolved: https://github.com/facebook/Ax/pull/2991 See D64969707 for context. This diff: * Moves the parts of `tests/runners/test_botorch_test_problem` that pertain to the runner into `tests/test_benchmark_runner.py` * Moves the parts that pertain to `BoTorchTestFunction` to `tests/benchmark_test_functions/test_botorch_test_function.py` * Moves `tests/runners/test_surrogate_runner.py` to `tests/benchmark_test_functions/test_surrogate_test_function.py` Reviewed By: Balandat Differential Revision: D65090663 fbshipit-source-id: 2f7dba23f3c149ab1e6a8c90c6eb48798660a4d9 --- .../test_botorch_test_function.py | 80 +++++++++++++++++++ .../test_surrogate_test_function.py} | 0 ax/benchmark/tests/runners/__init__.py | 7 -- ...st_problem.py => test_benchmark_runner.py} | 73 +---------------- 4 files changed, 83 insertions(+), 77 deletions(-) create mode 100644 ax/benchmark/tests/benchmark_test_functions/test_botorch_test_function.py rename ax/benchmark/tests/{runners/test_surrogate_runner.py => benchmark_test_functions/test_surrogate_test_function.py} (100%) delete mode 100644 ax/benchmark/tests/runners/__init__.py rename ax/benchmark/tests/{runners/test_botorch_test_problem.py => test_benchmark_runner.py} (78%) diff --git a/ax/benchmark/tests/benchmark_test_functions/test_botorch_test_function.py b/ax/benchmark/tests/benchmark_test_functions/test_botorch_test_function.py new file mode 100644 index 00000000000..f3c0136e713 --- /dev/null +++ b/ax/benchmark/tests/benchmark_test_functions/test_botorch_test_function.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-strict + + +import torch +from ax.benchmark.benchmark_test_functions.botorch_test import BoTorchTestFunction +from ax.utils.common.testutils import TestCase +from botorch.test_functions.multi_objective import BraninCurrin +from botorch.test_functions.synthetic import ConstrainedHartmann, Hartmann + + +class TestBoTorchTestFunction(TestCase): + def setUp(self) -> None: + super().setUp() + botorch_base_test_functions = { + "base Hartmann": Hartmann(dim=6), + "negated Hartmann": Hartmann(dim=6, negate=True), + "constrained Hartmann": ConstrainedHartmann(dim=6), + "negated constrained Hartmann": ConstrainedHartmann(dim=6, negate=True), + } + self.botorch_test_problems = { + k: BoTorchTestFunction(botorch_problem=v) + for k, v in botorch_base_test_functions.items() + } + + def test_negation(self) -> None: + params = {f"x{i}": 0.5 for i in range(6)} + evaluate_true_results = { + k: v.evaluate_true(params) for k, v in self.botorch_test_problems.items() + } + self.assertEqual( + evaluate_true_results["base Hartmann"], + evaluate_true_results["constrained Hartmann"][0], + ) + self.assertEqual( + evaluate_true_results["base Hartmann"], + -evaluate_true_results["negated Hartmann"], + ) + self.assertEqual( + evaluate_true_results["negated Hartmann"], + evaluate_true_results["negated constrained Hartmann"][0], + ) + self.assertEqual( + evaluate_true_results["constrained Hartmann"][1], + evaluate_true_results["negated constrained Hartmann"][1], + ) + + def test_raises_for_botorch_attrs(self) -> None: + msg = "noise should be set on the `BenchmarkRunner`, not the test function." + with self.assertRaisesRegex(ValueError, msg): + BoTorchTestFunction(botorch_problem=Hartmann(dim=6, noise_std=0.1)) + with self.assertRaisesRegex(ValueError, msg): + BoTorchTestFunction( + botorch_problem=ConstrainedHartmann(dim=6, constraint_noise_std=0.1) + ) + + def test_tensor_shapes(self) -> None: + params = {f"x{i}": 0.5 for i in range(6)} + evaluate_true_results = { + k: v.evaluate_true(params) for k, v in self.botorch_test_problems.items() + } + evaluate_true_results["BraninCurrin"] = BoTorchTestFunction( + botorch_problem=BraninCurrin() + ).evaluate_true(params) + expected_len = { + "base Hartmann": 1, + "constrained Hartmann": 2, + "negated Hartmann": 1, + "negated constrained Hartmann": 2, + "BraninCurrin": 2, + } + for name, result in evaluate_true_results.items(): + with self.subTest(name=name): + self.assertEqual(result.dtype, torch.double) + self.assertEqual(result.shape, torch.Size([expected_len[name]])) diff --git a/ax/benchmark/tests/runners/test_surrogate_runner.py b/ax/benchmark/tests/benchmark_test_functions/test_surrogate_test_function.py similarity index 100% rename from ax/benchmark/tests/runners/test_surrogate_runner.py rename to ax/benchmark/tests/benchmark_test_functions/test_surrogate_test_function.py diff --git a/ax/benchmark/tests/runners/__init__.py b/ax/benchmark/tests/runners/__init__.py deleted file mode 100644 index c412c1b3b4f..00000000000 --- a/ax/benchmark/tests/runners/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -# pyre-strict diff --git a/ax/benchmark/tests/runners/test_botorch_test_problem.py b/ax/benchmark/tests/test_benchmark_runner.py similarity index 78% rename from ax/benchmark/tests/runners/test_botorch_test_problem.py rename to ax/benchmark/tests/test_benchmark_runner.py index 1a88099ea97..f4e13aa6dbc 100644 --- a/ax/benchmark/tests/runners/test_botorch_test_problem.py +++ b/ax/benchmark/tests/test_benchmark_runner.py @@ -29,83 +29,16 @@ DummyTestFunction, get_soo_surrogate_test_function, ) -from botorch.test_functions.multi_objective import BraninCurrin from botorch.test_functions.synthetic import Ackley, ConstrainedHartmann, Hartmann from botorch.utils.transforms import normalize -class TestBoTorchTestFunction(TestCase): - def setUp(self) -> None: - super().setUp() - botorch_base_test_functions = { - "base Hartmann": Hartmann(dim=6), - "negated Hartmann": Hartmann(dim=6, negate=True), - "constrained Hartmann": ConstrainedHartmann(dim=6), - "negated constrained Hartmann": ConstrainedHartmann(dim=6, negate=True), - } - self.botorch_test_problems = { - k: BoTorchTestFunction(botorch_problem=v) - for k, v in botorch_base_test_functions.items() - } - - def test_negation(self) -> None: - params = {f"x{i}": 0.5 for i in range(6)} - evaluate_true_results = { - k: v.evaluate_true(params) for k, v in self.botorch_test_problems.items() - } - self.assertEqual( - evaluate_true_results["base Hartmann"], - evaluate_true_results["constrained Hartmann"][0], - ) - self.assertEqual( - evaluate_true_results["base Hartmann"], - -evaluate_true_results["negated Hartmann"], - ) - self.assertEqual( - evaluate_true_results["negated Hartmann"], - evaluate_true_results["negated constrained Hartmann"][0], - ) - self.assertEqual( - evaluate_true_results["constrained Hartmann"][1], - evaluate_true_results["negated constrained Hartmann"][1], - ) - - def test_raises_for_botorch_attrs(self) -> None: - msg = "noise should be set on the `BenchmarkRunner`, not the test function." - with self.assertRaisesRegex(ValueError, msg): - BoTorchTestFunction(botorch_problem=Hartmann(dim=6, noise_std=0.1)) - with self.assertRaisesRegex(ValueError, msg): - BoTorchTestFunction( - botorch_problem=ConstrainedHartmann(dim=6, constraint_noise_std=0.1) - ) - - def test_tensor_shapes(self) -> None: - params = {f"x{i}": 0.5 for i in range(6)} - evaluate_true_results = { - k: v.evaluate_true(params) for k, v in self.botorch_test_problems.items() - } - evaluate_true_results["BraninCurrin"] = BoTorchTestFunction( - botorch_problem=BraninCurrin() - ).evaluate_true(params) - expected_len = { - "base Hartmann": 1, - "constrained Hartmann": 2, - "negated Hartmann": 1, - "negated constrained Hartmann": 2, - "BraninCurrin": 2, - } - for name, result in evaluate_true_results.items(): - with self.subTest(name=name): - self.assertEqual(result.dtype, torch.double) - self.assertEqual(result.shape, torch.Size([expected_len[name]])) - - -class TestSyntheticRunner(TestCase): +class TestBenchmarkRunner(TestCase): def setUp(self) -> None: super().setUp() self.maxDiff = None - def test_synthetic_runner(self) -> None: + def test_runner(self) -> None: botorch_cases = [ ( BoTorchTestFunction( @@ -297,7 +230,7 @@ def test_synthetic_runner(self) -> None: ): BenchmarkRunner.deserialize_init_args({}) - def test_botorch_test_problem_runner_heterogeneous_noise(self) -> None: + def test_heterogeneous_noise(self) -> None: for noise_std in [[0.1, 0.05], {"objective": 0.1, "constraint": 0.05}]: runner = BenchmarkRunner( test_function=BoTorchTestFunction(