From b2d49408daca2fc8246793223527c7379b990a7b Mon Sep 17 00:00:00 2001 From: Elizabeth Santorella Date: Fri, 30 Aug 2024 13:25:15 -0700 Subject: [PATCH] remove unused metrics (#2728) Summary: Pull Request resolved: https://github.com/facebook/Ax/pull/2728 These may have once been used for benchmarking, but no longer are (not even in tests). Also, if these were to become needed again, it should be possible to define one metric based on BoTorch test functions that covers most of these, so we don't need so many classes that need their own registry entries. Also removed Sphinx headings for metrics files that aren't in Ax. * Removed Sphinx headings for BoTorch Test Problem and Dictionary Lookup, which don't exist anymore * Removed JenattonMetric, AugmentedBraninMetric, AugmentedHartmannMetric * Removed functionality from stubs that wasn't being used Reviewed By: saitcakmak Differential Revision: D61998254 fbshipit-source-id: 0be53742640520ead7ab6eba850a2612f47aacd1 --- ax/metrics/botorch_test_problem.py | 57 ------------------- ax/storage/json_store/registry.py | 9 +-- .../json_store/tests/test_json_store.py | 4 -- ax/utils/testing/core_stubs.py | 41 +------------ sphinx/source/metrics.rst | 25 -------- 5 files changed, 4 insertions(+), 132 deletions(-) delete mode 100644 ax/metrics/botorch_test_problem.py diff --git a/ax/metrics/botorch_test_problem.py b/ax/metrics/botorch_test_problem.py deleted file mode 100644 index 5660df35bf8..00000000000 --- a/ax/metrics/botorch_test_problem.py +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -# pyre-strict - -from typing import Any, Optional - -import pandas as pd -from ax.core.base_trial import BaseTrial -from ax.core.data import Data -from ax.core.metric import Metric, MetricFetchE, MetricFetchResult -from ax.utils.common.result import Err, Ok - - -class BotorchTestProblemMetric(Metric): - """A Metric for retriving information from a BotorchTestProblemRunner. - A BotorchTestProblemRunner will attach the result of a call to - BaseTestProblem.forward per Arm on a given trial, and this Metric will extract the - proper value from the resulting tensor given its index. - """ - - def __init__( - self, - name: str, - noise_sd: Optional[float] = None, - index: int = 0, - ) -> None: - super().__init__(name=name) - self.noise_sd = noise_sd - self.index = index - - def fetch_trial_data(self, trial: BaseTrial, **kwargs: Any) -> MetricFetchResult: - try: - mean = [ - trial.run_metadata["Ys"][name][self.index] - for name, arm in trial.arms_by_name.items() - ] - df = pd.DataFrame( - { - "arm_name": [name for name, _ in trial.arms_by_name.items()], - "metric_name": self.name, - "mean": mean, - # If no noise_std is returned then Botorch evaluated the true - # function - "sem": self.noise_sd, - "trial_index": trial.index, - } - ) - - return Ok(value=Data(df=df)) - - except Exception as e: - return Err( - MetricFetchE(message=f"Failed to fetch {self.name}", exception=e) - ) diff --git a/ax/storage/json_store/registry.py b/ax/storage/json_store/registry.py index 001d8f2a52e..be2d164a3b1 100644 --- a/ax/storage/json_store/registry.py +++ b/ax/storage/json_store/registry.py @@ -68,11 +68,11 @@ OrEarlyStoppingStrategy, ) from ax.global_stopping.strategies.improvement import ImprovementGlobalStoppingStrategy -from ax.metrics.branin import AugmentedBraninMetric, BraninMetric, NegativeBraninMetric +from ax.metrics.branin import BraninMetric, NegativeBraninMetric from ax.metrics.branin_map import BraninTimestampMapMetric from ax.metrics.chemistry import ChemistryMetric, ChemistryProblemType from ax.metrics.factorial import FactorialMetric -from ax.metrics.hartmann6 import AugmentedHartmann6Metric, Hartmann6Metric +from ax.metrics.hartmann6 import Hartmann6Metric from ax.metrics.l2norm import L2NormMetric from ax.metrics.noisy_function import NoisyFunctionMetric from ax.metrics.sklearn import SklearnDataset, SklearnMetric, SklearnModelType @@ -178,8 +178,6 @@ Arm: arm_to_dict, AuxiliaryExperiment: auxiliary_experiment_to_dict, AndEarlyStoppingStrategy: logical_early_stopping_strategy_to_dict, - AugmentedBraninMetric: metric_to_dict, - AugmentedHartmann6Metric: metric_to_dict, AutoTransitionAfterGen: transition_criterion_to_dict, BatchTrial: batch_to_dict, BenchmarkMetric: metric_to_dict, @@ -281,8 +279,6 @@ CORE_DECODER_REGISTRY: TDecoderRegistry = { "AbandonedArm": AbandonedArm, "AndEarlyStoppingStrategy": AndEarlyStoppingStrategy, - "AugmentedBraninMetric": AugmentedBraninMetric, - "AugmentedHartmann6Metric": AugmentedHartmann6Metric, "AutoTransitionAfterGen": AutoTransitionAfterGen, "AuxiliaryExperiment": AuxiliaryExperiment, "Arm": Arm, @@ -293,7 +289,6 @@ "BenchmarkProblem": BenchmarkProblem, "BenchmarkResult": BenchmarkResult, "BoTorchModel": BoTorchModel, - "BotorchTestProblemMetric": BenchmarkMetric, # backward-compatibility "BotorchTestProblemRunner": BotorchTestProblemRunner, "BraninMetric": BraninMetric, "BraninTimestampMapMetric": BraninTimestampMapMetric, diff --git a/ax/storage/json_store/tests/test_json_store.py b/ax/storage/json_store/tests/test_json_store.py index d6538ced2d0..68b2c95de51 100644 --- a/ax/storage/json_store/tests/test_json_store.py +++ b/ax/storage/json_store/tests/test_json_store.py @@ -60,8 +60,6 @@ get_acquisition_type, get_and_early_stopping_strategy, get_arm, - get_augmented_branin_metric, - get_augmented_hartmann_metric, get_auxiliary_experiment, get_batch_trial, get_botorch_model, @@ -142,8 +140,6 @@ ("AggregatedBenchmarkResult", get_aggregated_benchmark_result), ("AndEarlyStoppingStrategy", get_and_early_stopping_strategy), ("Arm", get_arm), - ("AugmentedBraninMetric", get_augmented_branin_metric), - ("AugmentedHartmannMetric", get_augmented_hartmann_metric), ("AuxiliaryExperiment", get_auxiliary_experiment), ("BatchTrial", get_batch_trial), ("BenchmarkMethod", get_sobol_gpei_benchmark_method), diff --git a/ax/utils/testing/core_stubs.py b/ax/utils/testing/core_stubs.py index 7923ebc3c76..2823ff7a6e6 100644 --- a/ax/utils/testing/core_stubs.py +++ b/ax/utils/testing/core_stubs.py @@ -80,10 +80,10 @@ from ax.exceptions.core import UserInputError from ax.global_stopping.strategies.base import BaseGlobalStoppingStrategy from ax.global_stopping.strategies.improvement import ImprovementGlobalStoppingStrategy -from ax.metrics.branin import AugmentedBraninMetric, BraninMetric +from ax.metrics.branin import BraninMetric from ax.metrics.branin_map import BraninTimestampMapMetric from ax.metrics.factorial import FactorialMetric -from ax.metrics.hartmann6 import AugmentedHartmann6Metric, Hartmann6Metric +from ax.metrics.hartmann6 import Hartmann6Metric from ax.modelbridge.factory import Cont_X_trans, get_factorial, get_sobol, Models from ax.modelbridge.generation_strategy import GenerationNode, GenerationStrategy from ax.modelbridge.model_spec import ModelSpec @@ -1494,24 +1494,11 @@ def get_branin_metric( ) -def get_augmented_branin_metric(name: str = "aug_branin") -> AugmentedBraninMetric: - param_names = ["x1", "x2", "fidelity"] - return AugmentedBraninMetric(name=name, param_names=param_names, noise_sd=0.01) - - def get_hartmann_metric(name: str = "hartmann") -> Hartmann6Metric: param_names = [f"x{idx + 1}" for idx in range(6)] return Hartmann6Metric(name=name, param_names=param_names, noise_sd=0.01) -def get_augmented_hartmann_metric( - name: str = "aug_hartmann", -) -> AugmentedHartmann6Metric: - param_names = [f"x{idx + 1}" for idx in range(6)] - param_names.append("fidelity") - return AugmentedHartmann6Metric(name=name, param_names=param_names, noise_sd=0.01) - - def get_factorial_metric(name: str = "success_metric") -> FactorialMetric: coefficients: dict[str, dict[TParamValue, float]] = { "factor1": {"level11": 0.1, "level12": 0.2, "level13": 0.3}, @@ -1635,18 +1622,6 @@ def get_branin_multi_objective(num_objectives: int = 2) -> MultiObjective: return MultiObjective(objectives=objectives) -def get_augmented_branin_objective() -> Objective: - return Objective(metric=get_augmented_branin_metric(), minimize=False) - - -def get_hartmann_objective() -> Objective: - return Objective(metric=get_hartmann_metric(), minimize=False) - - -def get_augmented_hartmann_objective() -> Objective: - return Objective(metric=get_augmented_hartmann_metric(), minimize=False) - - ############################## # Optimization Configs ############################## @@ -1745,18 +1720,6 @@ def get_branin_multi_objective_optimization_config( ) -def get_augmented_branin_optimization_config() -> OptimizationConfig: - return OptimizationConfig(objective=get_augmented_branin_objective()) - - -def get_hartmann_optimization_config() -> OptimizationConfig: - return OptimizationConfig(objective=get_hartmann_objective()) - - -def get_augmented_hartmann_optimization_config() -> OptimizationConfig: - return OptimizationConfig(objective=get_augmented_hartmann_objective()) - - ############################## # Arms ############################## diff --git a/sphinx/source/metrics.rst b/sphinx/source/metrics.rst index 82d90bd3b51..532ead03edc 100644 --- a/sphinx/source/metrics.rst +++ b/sphinx/source/metrics.rst @@ -8,14 +8,6 @@ ax.metrics .. currentmodule:: ax.metrics -BoTorch Test Problem -~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: ax.metrics.botorch_test_problem - :members: - :undoc-members: - :show-inheritance: - Branin ~~~~~~ @@ -48,14 +40,6 @@ Curve :undoc-members: :show-inheritance: -Dictionary Lookup -~~~~~~~~~~~~~~~~~ - -.. automodule:: ax.metrics.dict_lookup - :members: - :undoc-members: - :show-inheritance: - Factorial ~~~~~~~~~ @@ -72,15 +56,6 @@ Hartmann6 :undoc-members: :show-inheritance: -Jenatton -~~~~~~~~ - -.. automodule:: ax.metrics.jenatton - :members: - :undoc-members: - :show-inheritance: - - L2 Norm ~~~~~~~