From 1e20d512663f963e186ae08a01b6a3fb97988186 Mon Sep 17 00:00:00 2001 From: Elizabeth Santorella Date: Fri, 30 Aug 2024 08:21:46 -0700 Subject: [PATCH] remove unused metrics (#2728) Summary: Pull Request resolved: https://github.com/facebook/Ax/pull/2728 These may have once been used for benchmarking, but no longer are (not even in tests). Also, if these were to become needed again, it should be possible to define one metric based on BoTorch test functions that covers most of these, so we don't need so many classes that need their own registry entries. Also removed Sphinx headings for metrics files that aren't in Ax. * Removed Sphinx headings for BoTorch Test Problem and Dictionary Lookup, which don't exist anymore * Removed Hartmann6Metric, JenattonMetric, AugmentedBraninMetric, AugmentedhartmannMetric * Removed functionality from stubs that wasn't being used Differential Revision: D61998254 --- ax/metrics/botorch_test_problem.py | 57 ------------------- ax/storage/json_store/registry.py | 10 +--- .../json_store/tests/test_json_store.py | 4 -- ax/utils/testing/core_stubs.py | 45 +-------------- sphinx/source/metrics.rst | 25 -------- 5 files changed, 3 insertions(+), 138 deletions(-) delete mode 100644 ax/metrics/botorch_test_problem.py diff --git a/ax/metrics/botorch_test_problem.py b/ax/metrics/botorch_test_problem.py deleted file mode 100644 index 5660df35bf8..00000000000 --- a/ax/metrics/botorch_test_problem.py +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -# pyre-strict - -from typing import Any, Optional - -import pandas as pd -from ax.core.base_trial import BaseTrial -from ax.core.data import Data -from ax.core.metric import Metric, MetricFetchE, MetricFetchResult -from ax.utils.common.result import Err, Ok - - -class BotorchTestProblemMetric(Metric): - """A Metric for retriving information from a BotorchTestProblemRunner. - A BotorchTestProblemRunner will attach the result of a call to - BaseTestProblem.forward per Arm on a given trial, and this Metric will extract the - proper value from the resulting tensor given its index. - """ - - def __init__( - self, - name: str, - noise_sd: Optional[float] = None, - index: int = 0, - ) -> None: - super().__init__(name=name) - self.noise_sd = noise_sd - self.index = index - - def fetch_trial_data(self, trial: BaseTrial, **kwargs: Any) -> MetricFetchResult: - try: - mean = [ - trial.run_metadata["Ys"][name][self.index] - for name, arm in trial.arms_by_name.items() - ] - df = pd.DataFrame( - { - "arm_name": [name for name, _ in trial.arms_by_name.items()], - "metric_name": self.name, - "mean": mean, - # If no noise_std is returned then Botorch evaluated the true - # function - "sem": self.noise_sd, - "trial_index": trial.index, - } - ) - - return Ok(value=Data(df=df)) - - except Exception as e: - return Err( - MetricFetchE(message=f"Failed to fetch {self.name}", exception=e) - ) diff --git a/ax/storage/json_store/registry.py b/ax/storage/json_store/registry.py index 001d8f2a52e..f91f9910eeb 100644 --- a/ax/storage/json_store/registry.py +++ b/ax/storage/json_store/registry.py @@ -68,11 +68,11 @@ OrEarlyStoppingStrategy, ) from ax.global_stopping.strategies.improvement import ImprovementGlobalStoppingStrategy -from ax.metrics.branin import AugmentedBraninMetric, BraninMetric, NegativeBraninMetric +from ax.metrics.branin import BraninMetric, NegativeBraninMetric from ax.metrics.branin_map import BraninTimestampMapMetric from ax.metrics.chemistry import ChemistryMetric, ChemistryProblemType from ax.metrics.factorial import FactorialMetric -from ax.metrics.hartmann6 import AugmentedHartmann6Metric, Hartmann6Metric +from ax.metrics.hartmann6 import Hartmann6Metric from ax.metrics.l2norm import L2NormMetric from ax.metrics.noisy_function import NoisyFunctionMetric from ax.metrics.sklearn import SklearnDataset, SklearnMetric, SklearnModelType @@ -178,8 +178,6 @@ Arm: arm_to_dict, AuxiliaryExperiment: auxiliary_experiment_to_dict, AndEarlyStoppingStrategy: logical_early_stopping_strategy_to_dict, - AugmentedBraninMetric: metric_to_dict, - AugmentedHartmann6Metric: metric_to_dict, AutoTransitionAfterGen: transition_criterion_to_dict, BatchTrial: batch_to_dict, BenchmarkMetric: metric_to_dict, @@ -281,8 +279,6 @@ CORE_DECODER_REGISTRY: TDecoderRegistry = { "AbandonedArm": AbandonedArm, "AndEarlyStoppingStrategy": AndEarlyStoppingStrategy, - "AugmentedBraninMetric": AugmentedBraninMetric, - "AugmentedHartmann6Metric": AugmentedHartmann6Metric, "AutoTransitionAfterGen": AutoTransitionAfterGen, "AuxiliaryExperiment": AuxiliaryExperiment, "Arm": Arm, @@ -293,7 +289,6 @@ "BenchmarkProblem": BenchmarkProblem, "BenchmarkResult": BenchmarkResult, "BoTorchModel": BoTorchModel, - "BotorchTestProblemMetric": BenchmarkMetric, # backward-compatibility "BotorchTestProblemRunner": BotorchTestProblemRunner, "BraninMetric": BraninMetric, "BraninTimestampMapMetric": BraninTimestampMapMetric, @@ -314,7 +309,6 @@ "GenerationStep": GenerationStep, "GeneratorRun": GeneratorRun, "GeneratorRunStruct": GeneratorRunStruct, - "Hartmann6Metric": Hartmann6Metric, "HierarchicalSearchSpace": HierarchicalSearchSpace, "ImprovementGlobalStoppingStrategy": ImprovementGlobalStoppingStrategy, "Interval": Interval, diff --git a/ax/storage/json_store/tests/test_json_store.py b/ax/storage/json_store/tests/test_json_store.py index d6538ced2d0..68b2c95de51 100644 --- a/ax/storage/json_store/tests/test_json_store.py +++ b/ax/storage/json_store/tests/test_json_store.py @@ -60,8 +60,6 @@ get_acquisition_type, get_and_early_stopping_strategy, get_arm, - get_augmented_branin_metric, - get_augmented_hartmann_metric, get_auxiliary_experiment, get_batch_trial, get_botorch_model, @@ -142,8 +140,6 @@ ("AggregatedBenchmarkResult", get_aggregated_benchmark_result), ("AndEarlyStoppingStrategy", get_and_early_stopping_strategy), ("Arm", get_arm), - ("AugmentedBraninMetric", get_augmented_branin_metric), - ("AugmentedHartmannMetric", get_augmented_hartmann_metric), ("AuxiliaryExperiment", get_auxiliary_experiment), ("BatchTrial", get_batch_trial), ("BenchmarkMethod", get_sobol_gpei_benchmark_method), diff --git a/ax/utils/testing/core_stubs.py b/ax/utils/testing/core_stubs.py index 7923ebc3c76..a2d7e639e1c 100644 --- a/ax/utils/testing/core_stubs.py +++ b/ax/utils/testing/core_stubs.py @@ -80,10 +80,9 @@ from ax.exceptions.core import UserInputError from ax.global_stopping.strategies.base import BaseGlobalStoppingStrategy from ax.global_stopping.strategies.improvement import ImprovementGlobalStoppingStrategy -from ax.metrics.branin import AugmentedBraninMetric, BraninMetric +from ax.metrics.branin import BraninMetric from ax.metrics.branin_map import BraninTimestampMapMetric from ax.metrics.factorial import FactorialMetric -from ax.metrics.hartmann6 import AugmentedHartmann6Metric, Hartmann6Metric from ax.modelbridge.factory import Cont_X_trans, get_factorial, get_sobol, Models from ax.modelbridge.generation_strategy import GenerationNode, GenerationStrategy from ax.modelbridge.model_spec import ModelSpec @@ -1494,24 +1493,6 @@ def get_branin_metric( ) -def get_augmented_branin_metric(name: str = "aug_branin") -> AugmentedBraninMetric: - param_names = ["x1", "x2", "fidelity"] - return AugmentedBraninMetric(name=name, param_names=param_names, noise_sd=0.01) - - -def get_hartmann_metric(name: str = "hartmann") -> Hartmann6Metric: - param_names = [f"x{idx + 1}" for idx in range(6)] - return Hartmann6Metric(name=name, param_names=param_names, noise_sd=0.01) - - -def get_augmented_hartmann_metric( - name: str = "aug_hartmann", -) -> AugmentedHartmann6Metric: - param_names = [f"x{idx + 1}" for idx in range(6)] - param_names.append("fidelity") - return AugmentedHartmann6Metric(name=name, param_names=param_names, noise_sd=0.01) - - def get_factorial_metric(name: str = "success_metric") -> FactorialMetric: coefficients: dict[str, dict[TParamValue, float]] = { "factor1": {"level11": 0.1, "level12": 0.2, "level13": 0.3}, @@ -1635,18 +1616,6 @@ def get_branin_multi_objective(num_objectives: int = 2) -> MultiObjective: return MultiObjective(objectives=objectives) -def get_augmented_branin_objective() -> Objective: - return Objective(metric=get_augmented_branin_metric(), minimize=False) - - -def get_hartmann_objective() -> Objective: - return Objective(metric=get_hartmann_metric(), minimize=False) - - -def get_augmented_hartmann_objective() -> Objective: - return Objective(metric=get_augmented_hartmann_metric(), minimize=False) - - ############################## # Optimization Configs ############################## @@ -1745,18 +1714,6 @@ def get_branin_multi_objective_optimization_config( ) -def get_augmented_branin_optimization_config() -> OptimizationConfig: - return OptimizationConfig(objective=get_augmented_branin_objective()) - - -def get_hartmann_optimization_config() -> OptimizationConfig: - return OptimizationConfig(objective=get_hartmann_objective()) - - -def get_augmented_hartmann_optimization_config() -> OptimizationConfig: - return OptimizationConfig(objective=get_augmented_hartmann_objective()) - - ############################## # Arms ############################## diff --git a/sphinx/source/metrics.rst b/sphinx/source/metrics.rst index 82d90bd3b51..532ead03edc 100644 --- a/sphinx/source/metrics.rst +++ b/sphinx/source/metrics.rst @@ -8,14 +8,6 @@ ax.metrics .. currentmodule:: ax.metrics -BoTorch Test Problem -~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: ax.metrics.botorch_test_problem - :members: - :undoc-members: - :show-inheritance: - Branin ~~~~~~ @@ -48,14 +40,6 @@ Curve :undoc-members: :show-inheritance: -Dictionary Lookup -~~~~~~~~~~~~~~~~~ - -.. automodule:: ax.metrics.dict_lookup - :members: - :undoc-members: - :show-inheritance: - Factorial ~~~~~~~~~ @@ -72,15 +56,6 @@ Hartmann6 :undoc-members: :show-inheritance: -Jenatton -~~~~~~~~ - -.. automodule:: ax.metrics.jenatton - :members: - :undoc-members: - :show-inheritance: - - L2 Norm ~~~~~~~