From a78ef2288b74b34b1f176d1fe3f0caca931c1902 Mon Sep 17 00:00:00 2001 From: Elizabeth Santorella Date: Wed, 24 Jul 2024 14:52:37 -0700 Subject: [PATCH] Rename `maximum_hypervolume` to `optimal_value` in MOO Benchmark Problems Summary: Context: This will give single-objective and multi-objective benchmark problems a more consistent API, making it possible to simplify the inheritance structure in D60145193 and subsequent PRs. Note: This breaks backwards compatibility, including for reading multi-objective benchmark problems stored as JSON. I think it's worthwhile because this will enable a lot of simplification in the future and because I expect we will want to break backward-compatibility again shortly. It would be possible to make this change BC, but I'm not sure it's worthwhile. Note: Docstrings are updated in D60145193, so they are not updated here. This PR: * Makes `MultiObjectiveBenchmarkProblem.__init__` take an `optimal_value` argument, and removes its `maximum_hypervolume` argument. * Removes the `optimal_value` property of `MultiObjectiveBenchmarkProblem`, which pointed to `maximum_hypervolume`. * Does the same for surrogate benchmark problems. * Updates references to `maximum_hypervolume`. Differential Revision: D60194654 --- ax/benchmark/benchmark_problem.py | 22 +++++++++-------- ax/benchmark/problems/surrogate.py | 26 ++++++++------------ ax/benchmark/tests/test_benchmark_problem.py | 4 +-- ax/storage/json_store/encoders.py | 2 +- ax/utils/testing/benchmark_stubs.py | 2 +- 5 files changed, 25 insertions(+), 31 deletions(-) diff --git a/ax/benchmark/benchmark_problem.py b/ax/benchmark/benchmark_problem.py index c2c60ccd717..4eecdfddf43 100644 --- a/ax/benchmark/benchmark_problem.py +++ b/ax/benchmark/benchmark_problem.py @@ -314,14 +314,16 @@ def from_botorch_synthetic( class MultiObjectiveBenchmarkProblem(BenchmarkProblem): - """A BenchmarkProblem support multiple objectives. Rather than knowing each - objective's optimal value we track a known maximum hypervolume computed from a - given reference point. + """ + A `BenchmarkProblem` that supports multiple objectives. + + For multi-objective problems, `optimal_value` indicates the maximum + hypervolume attainable with the given `reference_point`. """ def __init__( self, - maximum_hypervolume: float, + optimal_value: float, reference_point: List[float], *, name: str, @@ -334,7 +336,7 @@ def __init__( has_ground_truth: bool = False, tracking_metrics: Optional[List[BenchmarkMetricBase]] = None, ) -> None: - self.maximum_hypervolume = maximum_hypervolume + self.optimal_value = optimal_value self.reference_point = reference_point super().__init__( name=name, @@ -348,10 +350,6 @@ def __init__( tracking_metrics=tracking_metrics, ) - @property - def optimal_value(self) -> float: - return self.maximum_hypervolume - @classmethod def from_botorch_multi_objective( cls, @@ -425,6 +423,10 @@ def from_botorch_multi_objective( is_noiseless=problem.is_noiseless, observe_noise_sd=observe_noise_sd, has_ground_truth=problem.has_ground_truth, - maximum_hypervolume=test_problem.max_hv, + optimal_value=test_problem.max_hv, reference_point=test_problem._ref_point, ) + + @property + def maximum_hypervolume(self) -> float: + return self.optimal_value diff --git a/ax/benchmark/problems/surrogate.py b/ax/benchmark/problems/surrogate.py index 3b07a57d81f..4f08ecb84f8 100644 --- a/ax/benchmark/problems/surrogate.py +++ b/ax/benchmark/problems/surrogate.py @@ -27,8 +27,8 @@ class SurrogateBenchmarkProblemBase(Base): """ Base class for SOOSurrogateBenchmarkProblem and MOOSurrogateBenchmarkProblem. - Allows for lazy creation of objects needed to construct a `runner`, - including a surrogate and datasets. + Its `runner` is created lazily, when `runner` is accessed or `set_runner` is + called, to defer construction of the surrogate and downloading of datasets. """ def __init__( @@ -147,8 +147,10 @@ def __repr__(self) -> str: class SOOSurrogateBenchmarkProblem(SurrogateBenchmarkProblemBase): """ - Has the same attributes/properties as a `SingleObjectiveBenchmarkProblem`, - but allows for constructing from a surrogate. + Has the same attributes/properties as a `MultiObjectiveBenchmarkProblem`, + but its runner is not constructed until needed, to allow for deferring + constructing the surrogate and downloading data. The surrogate is only + defined when `runner` is accessed or `set_runner` is called. """ def __init__( @@ -187,19 +189,15 @@ class MOOSurrogateBenchmarkProblem(SurrogateBenchmarkProblemBase): """ Has the same attributes/properties as a `MultiObjectiveBenchmarkProblem`, but its runner is not constructed until needed, to allow for deferring - constructing the surrogate. - - Simple aspects of the problem problem such as its search space - are defined immediately, while the surrogate is only defined when [TODO] - in order to avoid expensive operations like downloading files and fitting - a model. + constructing the surrogate and downloading data. The surrogate is only + defined when `runner` is accessed or `set_runner` is called. """ optimization_config: MultiObjectiveOptimizationConfig def __init__( self, - maximum_hypervolume: float, + optimal_value: float, reference_point: List[float], *, name: str, @@ -228,8 +226,4 @@ def __init__( _runner=_runner, ) self.reference_point = reference_point - self.maximum_hypervolume = maximum_hypervolume - - @property - def optimal_value(self) -> float: - return self.maximum_hypervolume + self.optimal_value = optimal_value diff --git a/ax/benchmark/tests/test_benchmark_problem.py b/ax/benchmark/tests/test_benchmark_problem.py index d0b9f9ec6b7..a0ac38153c1 100644 --- a/ax/benchmark/tests/test_benchmark_problem.py +++ b/ax/benchmark/tests/test_benchmark_problem.py @@ -194,9 +194,7 @@ def test_moo_from_botorch(self) -> None: ) # Test hypervolume - self.assertEqual( - branin_currin_problem.maximum_hypervolume, test_problem._max_hv - ) + self.assertEqual(branin_currin_problem.optimal_value, test_problem._max_hv) self.assertEqual(branin_currin_problem.reference_point, test_problem._ref_point) def test_maximization_problem(self) -> None: diff --git a/ax/storage/json_store/encoders.py b/ax/storage/json_store/encoders.py index e496617b7af..08045bec85c 100644 --- a/ax/storage/json_store/encoders.py +++ b/ax/storage/json_store/encoders.py @@ -165,7 +165,7 @@ def multi_objective_benchmark_problem_to_dict( "observe_noise_sd": moo_benchmark_problem.observe_noise_sd, "has_ground_truth": moo_benchmark_problem.has_ground_truth, "tracking_metrics": moo_benchmark_problem.tracking_metrics, - "maximum_hypervolume": moo_benchmark_problem.maximum_hypervolume, + "optimal_value": moo_benchmark_problem.optimal_value, "reference_point": moo_benchmark_problem.reference_point, } diff --git a/ax/utils/testing/benchmark_stubs.py b/ax/utils/testing/benchmark_stubs.py index 8aa787a596e..170334b0258 100644 --- a/ax/utils/testing/benchmark_stubs.py +++ b/ax/utils/testing/benchmark_stubs.py @@ -143,7 +143,7 @@ def get_moo_surrogate() -> MOOSurrogateBenchmarkProblem: outcome_names=["branin_a", "branin_b"], observe_noise_stds=True, get_surrogate_and_datasets=lambda: (surrogate, []), - maximum_hypervolume=1.0, + optimal_value=1.0, reference_point=[], )