Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Remove functionality for BenchmarkRunner without ground truth #2674

Closed
wants to merge 1 commit into from
Closed

Remove functionality for BenchmarkRunner without ground truth #2674

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
106 changes: 49 additions & 57 deletions ax/benchmark/runners/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,9 @@

# pyre-strict

from abc import ABC, abstractmethod
from abc import ABC, abstractmethod, abstractproperty
from math import sqrt
from typing import Any, Optional, Union
from typing import Any, Union

import torch
from ax.core.arm import Arm
Expand All @@ -21,45 +21,44 @@


class BenchmarkRunner(Runner, ABC):

@property
@abstractmethod
"""
A Runner that produces both observed and ground-truth values.

Observed values equal ground-truth values plus noise, with the noise added
according to the standard deviations returned by `get_noise_stds()`.

This runner does require that every benchmark has a ground truth, which
won't necessarily be true for real-world problems. Such problems fall into
two categories:
- If they are deterministic, they can be used with this runner by
viewing them as noiseless problems where the observed values are the
ground truth. The observed values will be used for tracking the
progress of optimization.
- If they are not deterministc, they are not supported. It is not
conceptually clear how to benchmark such problems, so we decided to
not over-engineer for that before such a use case arrives.
"""

@abstractproperty
def outcome_names(self) -> list[str]:
"""The names of the outcomes of the problem (in the order of the outcomes)."""
pass # pragma: no cover

def get_Y_true(self, arm: Arm) -> Tensor:
"""Function returning the ground truth values for a given arm. The
synthetic noise is added as part of the Runner's `run()` method.
For problems that do not have a ground truth, the Runner must
implement the `get_Y_Ystd()` method instead."""
raise NotImplementedError(
"Must implement method `get_Y_true()` for Runner "
f"{self.__class__.__name__} as it does not implement a "
"`get_Y_Ystd()` method."
)
"""
Return the ground truth values for a given arm.

Synthetic noise is added as part of the Runner's `run()` method.
"""
...

@abstractmethod
def get_noise_stds(self) -> Union[None, float, dict[str, float]]:
"""Function returning the standard errors for the synthetic noise
to be applied to the observed values. For problems that do not have
a ground truth, the Runner must implement the `get_Y_Ystd()` method
instead."""
raise NotImplementedError(
"Must implement method `get_Y_Ystd()` for Runner "
f"{self.__class__.__name__} as it does not implement a "
"`get_noise_stds()` method."
)

def get_Y_Ystd(self, arm: Arm) -> tuple[Tensor, Optional[Tensor]]:
"""Function returning the observed values and their standard errors
for a given arm. This function is unused for problems that have a
ground truth (in this case `get_Y_true()` is used), and is required
for problems that do not have a ground truth."""
raise NotImplementedError(
"Must implement method `get_Y_Ystd()` for Runner "
f"{self.__class__.__name__} as it does not implement a "
"`get_Y_true()` method."
)
"""
Return the standard errors for the synthetic noise to be applied to the
observed values.
"""
...

def run(self, trial: BaseTrial) -> dict[str, Any]:
"""Run the trial by evaluating its parameterization(s).
Expand Down Expand Up @@ -110,33 +109,26 @@ def run(self, trial: BaseTrial) -> dict[str, Any]:
)

for arm in trial.arms:
try:
# Case where we do have a ground truth
Y_true = self.get_Y_true(arm)
Ys_true[arm.name] = Y_true.tolist()
if noise_stds is None:
# No noise, so just return the true outcome.
Ystds[arm.name] = [0.0] * len(Y_true)
Ys[arm.name] = Y_true.tolist()
else:
# We can scale the noise std by the inverse of the relative sample
# budget allocation to each arm. This works b/c (i) we assume that
# observations per unit sample budget are i.i.d. and (ii) the
# normalized weights sum to one.
std = noise_stds_tsr.to(Y_true) / sqrt(nlzd_arm_weights[arm])
Ystds[arm.name] = std.tolist()
Ys[arm.name] = (Y_true + std * torch.randn_like(Y_true)).tolist()
except NotImplementedError:
# Case where we don't have a ground truth.
Y, Ystd = self.get_Y_Ystd(arm)
Ys[arm.name] = Y.tolist()
Ystds[arm.name] = Ystd.tolist() if Ystd is not None else None
# Case where we do have a ground truth
Y_true = self.get_Y_true(arm)
Ys_true[arm.name] = Y_true.tolist()
if noise_stds is None:
# No noise, so just return the true outcome.
Ystds[arm.name] = [0.0] * len(Y_true)
Ys[arm.name] = Y_true.tolist()
else:
# We can scale the noise std by the inverse of the relative sample
# budget allocation to each arm. This works b/c (i) we assume that
# observations per unit sample budget are i.i.d. and (ii) the
# normalized weights sum to one.
std = noise_stds_tsr.to(Y_true) / sqrt(nlzd_arm_weights[arm])
Ystds[arm.name] = std.tolist()
Ys[arm.name] = (Y_true + std * torch.randn_like(Y_true)).tolist()

run_metadata = {
"Ys": Ys,
"Ystds": Ystds,
"outcome_names": self.outcome_names,
"Ys_true": Ys_true,
}
if Ys_true: # only add key if we actually have a ground truth
run_metadata["Ys_true"] = Ys_true
return run_metadata