From 9571fab915f79fa35ec9f5cd50b609dcbb253181 Mon Sep 17 00:00:00 2001 From: Lisa Dunlap Date: Thu, 1 Aug 2019 19:52:08 -0700 Subject: [PATCH 01/31] tabular benchmarks --- python/ray/tune/examples/FCNet.py | 196 +++++++++++++++ python/ray/tune/examples/FCNet_Trainable.py | 157 ++++++++++++ python/ray/tune/examples/NASCifar10.py | 226 ++++++++++++++++++ .../ray/tune/examples/NASCifar_Trainable.py | 100 ++++++++ 4 files changed, 679 insertions(+) create mode 100644 python/ray/tune/examples/FCNet.py create mode 100644 python/ray/tune/examples/FCNet_Trainable.py create mode 100644 python/ray/tune/examples/NASCifar10.py create mode 100644 python/ray/tune/examples/NASCifar_Trainable.py diff --git a/python/ray/tune/examples/FCNet.py b/python/ray/tune/examples/FCNet.py new file mode 100644 index 000000000000..3d7afafd6402 --- /dev/null +++ b/python/ray/tune/examples/FCNet.py @@ -0,0 +1,196 @@ +""" Code from https://github.com/automl/nas_benchmarks """ + +import json +import os + +import ConfigSpace +import h5py +import numpy as np + + +class FCNetBenchmark(object): + + def __init__(self, path, dataset="fcnet_protein_structure_data.hdf5", seed=None): + + cs = self.get_configuration_space() + self.names = [h.name for h in cs.get_hyperparameters()] + + self.data = h5py.File(os.path.join(path, dataset), "r") + + self.X = [] + self.y = [] + self.c = [] + + self.rng = np.random.RandomState(seed) + + def get_best_configuration(self): + + """ + Returns the best configuration in the dataset that achieves the lowest test performance. + + :return: Returns tuple with the best configuration, its final validation performance and its test performance + """ + + configs, te, ve = [], [], [] + for k in self.data.keys(): + configs.append(json.loads(k)) + te.append(np.mean(self.data[k]["final_test_error"])) + ve.append(np.mean(self.data[k]["valid_mse"][:, -1])) + + b = np.argmin(te) + + return configs[b], ve[b], te[b] + + def objective_function(self, config, budget=100, **kwargs): + + assert 0 < budget <= 100 # check whether budget is in the correct bounds + + i = self.rng.randint(4) + + if type(config) == ConfigSpace.Configuration: + k = json.dumps(config.get_dictionary(), sort_keys=True) + else: + k = json.dumps(config, sort_keys=True) + + valid = self.data[k]["valid_mse"][i] + runtime = self.data[k]["runtime"][i] + + time_per_epoch = runtime / 100 # divide by the maximum number of epochs + + rt = time_per_epoch * budget + + self.X.append(config) + self.y.append(valid[budget - 1]) + self.c.append(rt) + + return valid[budget - 1], rt + + def objective_function_learning_curve(self, config, budget=100): + + assert 0 < budget <= 100 # check whether budget is in the correct bounds + + index = self.rng.randint(4) + + if type(config) == ConfigSpace.Configuration: + k = json.dumps(config.get_dictionary(), sort_keys=True) + else: + k = json.dumps(config, sort_keys=True) + + lc = [self.data[k]["valid_mse"][index][i] for i in range(budget)] + runtime = self.data[k]["runtime"][index] + + time_per_epoch = runtime / 100 # divide by the maximum number of epochs + + rt = [time_per_epoch * (i + 1) for i in range(budget)] + + self.X.append(config) + self.y.append(lc[-1]) + self.c.append(rt[-1]) + + return lc, rt + + def objective_function_deterministic(self, config, budget=100, index=0, **kwargs): + + assert 0 < budget <= 100 # check whether budget is in the correct bounds + + if type(config) == ConfigSpace.Configuration: + k = json.dumps(config.get_dictionary(), sort_keys=True) + else: + k = json.dumps(config, sort_keys=True) + + valid = self.data[k]["valid_mse"][index] + runtime = self.data[k]["runtime"][index] + + time_per_epoch = runtime / 100 # divide by the maximum number of epochs + + rt = time_per_epoch * budget + + self.X.append(config) + self.y.append(valid[budget - 1]) + self.c.append(rt) + + return valid[budget - 1], rt + + def objective_function_test(self, config, **kwargs): + + if type(config) == ConfigSpace.Configuration: + k = json.dumps(config.get_dictionary(), sort_keys=True) + else: + k = json.dumps(config, sort_keys=True) + + test = np.mean(self.data[k]["final_test_error"]) + runtime = np.mean(self.data[k]["runtime"]) + + return test, runtime + + def get_results(self): + + inc, y_star_valid, y_star_test = self.get_best_configuration() + + regret_validation = [] + regret_test = [] + runtime = [] + rt = 0 + + inc_valid = np.inf + inc_test = np.inf + + for i in range(len(self.X)): + + if inc_valid > self.y[i]: + inc_valid = self.y[i] + inc_test, _ = self.objective_function_test(self.X[i]) + + regret_validation.append(float(inc_valid - y_star_valid)) + regret_test.append(float(inc_test - y_star_test)) + rt += self.c[i] + runtime.append(float(rt)) + + res = dict() + res['regret_validation'] = regret_validation + res['regret_test'] = regret_test + res['runtime'] = runtime + + return res + + @staticmethod + def get_configuration_space(): + cs = ConfigSpace.ConfigurationSpace() + + cs.add_hyperparameter(ConfigSpace.OrdinalHyperparameter("n_units_1", [16, 32, 64, 128, 256, 512])) + cs.add_hyperparameter(ConfigSpace.OrdinalHyperparameter("n_units_2", [16, 32, 64, 128, 256, 512])) + cs.add_hyperparameter(ConfigSpace.OrdinalHyperparameter("dropout_1", [0.0, 0.3, 0.6])) + cs.add_hyperparameter(ConfigSpace.OrdinalHyperparameter("dropout_2", [0.0, 0.3, 0.6])) + cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("activation_fn_1", ["tanh", "relu"])) + cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("activation_fn_2", ["tanh", "relu"])) + cs.add_hyperparameter( + ConfigSpace.OrdinalHyperparameter("init_lr", [5 * 1e-4, 1e-3, 5 * 1e-3, 1e-2, 5 * 1e-2, 1e-1])) + cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("lr_schedule", ["cosine", "const"])) + cs.add_hyperparameter(ConfigSpace.OrdinalHyperparameter("batch_size", [8, 16, 32, 64])) + return cs + + +class FCNetSliceLocalizationBenchmark(FCNetBenchmark): + + def __init__(self, data_dir="./"): + super(FCNetSliceLocalizationBenchmark, self).__init__(path=data_dir, + dataset="fcnet_slice_localization_data.hdf5") + + +class FCNetProteinStructureBenchmark(FCNetBenchmark): + + def __init__(self, data_dir="./"): + super(FCNetProteinStructureBenchmark, self).__init__(path=data_dir, dataset="fcnet_protein_structure_data.hdf5") + + +class FCNetNavalPropulsionBenchmark(FCNetBenchmark): + + def __init__(self, data_dir="./"): + super(FCNetNavalPropulsionBenchmark, self).__init__(path=data_dir, dataset="fcnet_naval_propulsion_data.hdf5") + + +class FCNetParkinsonsTelemonitoringBenchmark(FCNetBenchmark): + + def __init__(self, data_dir="./"): + super(FCNetParkinsonsTelemonitoringBenchmark, self).__init__(path=data_dir, + dataset="fcnet_parkinsons_telemonitoring_data.hdf5") \ No newline at end of file diff --git a/python/ray/tune/examples/FCNet_Trainable.py b/python/ray/tune/examples/FCNet_Trainable.py new file mode 100644 index 000000000000..6372f1d59aba --- /dev/null +++ b/python/ray/tune/examples/FCNet_Trainable.py @@ -0,0 +1,157 @@ +""" Trainable using FCNet Benchmark from Tabular Benchmarks for Hyperparameter Optimization and Neural Architecture Search """ +from __future__ import print_function + +import os + +from ray.tune import Trainable +import tarfile +import urllib +from FCNet import FCNetBenchmark, FCNetProteinStructureBenchmark, FCNetSliceLocalizationBenchmark, FCNetNavalPropulsionBenchmark, FCNetParkinsonsTelemonitoringBenchmark + + +class FCNetProteinStructureTrainable(Trainable): + def __init__(self, config=None, logger_creator=None): + + # download dataset + file_tmp = urllib.urlretrieve('http://ml4aad.org/wp-content/uploads/2019/01/fcnet_tabular_benchmarks.tar.gz', filename=None)[0] + base_name = os.path.basename('http://ml4aad.org/wp-content/uploads/2019/01/fcnet_tabular_benchmarks.tar.gz') + + file_name, file_extension = os.path.splitext(base_name) + tar = tarfile.open(file_tmp) + tar.extractall(file_name) + + self._global_start = config.get("start", time.time()) + self._trial_start = time.time() + self.config = config + cwd = os.getcwd() + super(FCNetProteinStructureTrainable, self).__init__(config, logger_creator) + if ray.worker._mode() == ray.worker.LOCAL_MODE: + os.chdir(cwd) + self.net =FCNetProteinStructureBenchmark() + self.iteration = 0 + + def _train(self): + acc, time = self.net.objective_function(self.config, self.iteration) + self.iteration += 1 + return {"validation_accuracy": acc, 'runtime': time} + + def _save(self, checkpoint_dir): + path = os.path.join(checkpoint_dir, "checkpoint") + with open(path, "w") as f: + f.write(json.dumps({"iteration": self.iteration})) + return path + + def _restore(self, checkpoint_path): + with open(checkpoint_path) as f: + self.iteration = json.loads(f.read())["iteration"] + + +class FCNetSliceLocalizationTrainable(Trainable): + def __init__(self, config=None, logger_creator=None): + + # download dataset + file_tmp = urllib.urlretrieve('http://ml4aad.org/wp-content/uploads/2019/01/fcnet_tabular_benchmarks.tar.gz', filename=None)[0] + base_name = os.path.basename('http://ml4aad.org/wp-content/uploads/2019/01/fcnet_tabular_benchmarks.tar.gz') + + file_name, file_extension = os.path.splitext(base_name) + tar = tarfile.open(file_tmp) + tar.extractall(file_name) + + self._global_start = config.get("start", time.time()) + self._trial_start = time.time() + self.config = config + cwd = os.getcwd() + super(FCNetSliceLocalizationTrainable, self).__init__(config, logger_creator) + if ray.worker._mode() == ray.worker.LOCAL_MODE: + os.chdir(cwd) + self.net =FCNetProteinStructureBenchmark() + self.iteration = 0 + + def _train(self): + acc, time = self.net.objective_function(self.config, self.iteration) + self.iteration += 1 + return {"validation_accuracy": acc} + + def _save(self, checkpoint_dir): + path = os.path.join(checkpoint_dir, "checkpoint") + with open(path, "w") as f: + f.write(json.dumps({"iteration": self.iteration})) + return path + + def _restore(self, checkpoint_path): + with open(checkpoint_path) as f: + self.iteration = json.loads(f.read())["iteration"] + + +class FCNetNavalPropulsionTrainable(Trainable): + def __init__(self, config=None, logger_creator=None): + + # download dataset + file_tmp = urllib.urlretrieve('http://ml4aad.org/wp-content/uploads/2019/01/fcnet_tabular_benchmarks.tar.gz', filename=None)[0] + base_name = os.path.basename('http://ml4aad.org/wp-content/uploads/2019/01/fcnet_tabular_benchmarks.tar.gz') + + file_name, file_extension = os.path.splitext(base_name) + tar = tarfile.open(file_tmp) + tar.extractall(file_name) + + self._global_start = config.get("start", time.time()) + self._trial_start = time.time() + self.config = config + cwd = os.getcwd() + super(FCNetNavalPropulsionTrainable, self).__init__(config, logger_creator) + if ray.worker._mode() == ray.worker.LOCAL_MODE: + os.chdir(cwd) + self.net =FCNetProteinStructureBenchmark() + self.iteration = 0 + + def _train(self): + acc, time = self.net.objective_function(self.config, self.iteration) + self.iteration += 1 + return {"validation_accuracy": acc} + + def _save(self, checkpoint_dir): + path = os.path.join(checkpoint_dir, "checkpoint") + with open(path, "w") as f: + f.write(json.dumps({"iteration": self.iteration})) + return path + + def _restore(self, checkpoint_path): + with open(checkpoint_path) as f: + self.iteration = json.loads(f.read())["iteration"] + + +class FCNetParkinsonsTelemonitoringTrainable(Trainable): + def __init__(self, config=None, logger_creator=None): + + # download dataset + file_tmp = urllib.urlretrieve('http://ml4aad.org/wp-content/uploads/2019/01/fcnet_tabular_benchmarks.tar.gz', filename=None)[0] + base_name = os.path.basename('http://ml4aad.org/wp-content/uploads/2019/01/fcnet_tabular_benchmarks.tar.gz') + + file_name, file_extension = os.path.splitext(base_name) + tar = tarfile.open(file_tmp) + tar.extractall(file_name) + + self._global_start = config.get("start", time.time()) + self._trial_start = time.time() + self.config = config + cwd = os.getcwd() + super(FCNetParkinsonsTelemonitoringTrainable, self).__init__(config, logger_creator) + if ray.worker._mode() == ray.worker.LOCAL_MODE: + os.chdir(cwd) + self.net =FCNetProteinStructureBenchmark() + self.iteration = 0 + + def _train(self): + acc, time = self.net.objective_function(self.config, self.iteration) + self.iteration += 1 + return {"validation_accuracy": acc} + + def _save(self, checkpoint_dir): + path = os.path.join(checkpoint_dir, "checkpoint") + with open(path, "w") as f: + f.write(json.dumps({"iteration": self.iteration})) + return path + + def _restore(self, checkpoint_path): + with open(checkpoint_path) as f: + self.iteration = json.loads(f.read())["iteration"] \ No newline at end of file diff --git a/python/ray/tune/examples/NASCifar10.py b/python/ray/tune/examples/NASCifar10.py new file mode 100644 index 000000000000..6ee3b32a3bf4 --- /dev/null +++ b/python/ray/tune/examples/NASCifar10.py @@ -0,0 +1,226 @@ +""" Code from https://github.com/automl/nas_benchmarks """ + +import os + +import ConfigSpace +import numpy as np +from nasbench import api +from nasbench.lib import graph_util + +MAX_EDGES = 9 +VERTICES = 7 + + +class NASCifar10(object): + + def __init__(self, data_dir): + + self.dataset = api.NASBench(os.path.join(data_dir, 'nasbench_full.tfrecord')) + self.X = [] + self.y_valid = [] + self.y_test = [] + self.costs = [] + + self.y_star_valid = 0.04944576819737756 # lowest mean validation error + self.y_star_test = 0.056824247042338016 # lowest mean test error + + @staticmethod + def objective_function(self, config): + pass + + def record_invalid(self, config, valid, test, costs): + self.X.append(config) + self.y_valid.append(valid) + self.y_test.append(test) + self.costs.append(costs) + + def record_valid(self, config, data, model_spec): + + self.X.append(config) + + # compute mean test error for the final budget + _, metrics = self.dataset.get_metrics_from_spec(model_spec) + mean_test_error = 1 - np.mean([metrics[108][i]["final_test_accuracy"] for i in range(3)]) + self.y_test.append(mean_test_error) + + # compute validation error for the chosen budget + valid_error = 1 - data["validation_accuracy"] + self.y_valid.append(valid_error) + + runtime = data["training_time"] + self.costs.append(runtime) + + @staticmethod + def get_configuration_space(): + pass + + def get_results(self, ignore_invalid_configs=False): + + regret_validation = [] + regret_test = [] + runtime = [] + rt = 0 + + inc_valid = np.inf + inc_test = np.inf + + for i in range(len(self.X)): + + if ignore_invalid_configs and self.costs[i] == 0: + continue + + if inc_valid > self.y_valid[i]: + inc_valid = self.y_valid[i] + inc_test = self.y_test[i] + + regret_validation.append(float(inc_valid - self.y_star_valid)) + regret_test.append(float(inc_test - self.y_star_test)) + rt += self.costs[i] + runtime.append(float(rt)) + + res = dict() + res['regret_validation'] = regret_validation + res['regret_test'] = regret_test + res['runtime'] = runtime + + return res + + +class NASCifar10A(NASCifar10): + def objective_function(self, config, budget=108): + matrix = np.zeros([VERTICES, VERTICES], dtype=np.int8) + idx = np.triu_indices(matrix.shape[0], k=1) + for i in range(VERTICES * (VERTICES - 1) // 2): + row = idx[0][i] + col = idx[1][i] + matrix[row, col] = config["edge_%d" % i] + + # if not graph_util.is_full_dag(matrix) or graph_util.num_edges(matrix) > MAX_EDGES: + if graph_util.num_edges(matrix) > MAX_EDGES: + self.record_invalid(config, 1, 1, 0) + return 1, 0 + + labeling = [config["op_node_%d" % i] for i in range(5)] + labeling = ['input'] + list(labeling) + ['output'] + model_spec = api.ModelSpec(matrix, labeling) + try: + data = self.dataset.query(model_spec, epochs=budget) + except api.OutOfDomainError: + self.record_invalid(config, 1, 1, 0) + return 1, 0 + + self.record_valid(config, data, model_spec) + return 1 - data["validation_accuracy"], data["training_time"] + + @staticmethod + def get_configuration_space(): + cs = ConfigSpace.ConfigurationSpace() + + ops_choices = ['conv1x1-bn-relu', 'conv3x3-bn-relu', 'maxpool3x3'] + cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("op_node_0", ops_choices)) + cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("op_node_1", ops_choices)) + cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("op_node_2", ops_choices)) + cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("op_node_3", ops_choices)) + cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("op_node_4", ops_choices)) + for i in range(VERTICES * (VERTICES - 1) // 2): + cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("edge_%d" % i, [0, 1])) + return cs + + +class NASCifar10B(NASCifar10): + def objective_function(self, config, budget=108): + + bitlist = [0] * (VERTICES * (VERTICES - 1) // 2) + for i in range(MAX_EDGES): + bitlist[config["edge_%d" % i]] = 1 + out = 0 + for bit in bitlist: + out = (out << 1) | bit + + matrix = np.fromfunction(graph_util.gen_is_edge_fn(out), + (VERTICES, VERTICES), + dtype=np.int8) + # if not graph_util.is_full_dag(matrix) or graph_util.num_edges(matrix) > MAX_EDGES: + if graph_util.num_edges(matrix) > MAX_EDGES: + self.record_invalid(config, 1, 1, 0) + return 1, 0 + + labeling = [config["op_node_%d" % i] for i in range(5)] + labeling = ['input'] + list(labeling) + ['output'] + model_spec = api.ModelSpec(matrix, labeling) + try: + data = self.dataset.query(model_spec, epochs=budget) + except api.OutOfDomainError: + self.record_invalid(config, 1, 1, 0) + return 1, 0 + + self.record_valid(config, data, model_spec) + + return 1 - data["validation_accuracy"], data["training_time"] + + @staticmethod + def get_configuration_space(): + cs = ConfigSpace.ConfigurationSpace() + + ops_choices = ['conv1x1-bn-relu', 'conv3x3-bn-relu', 'maxpool3x3'] + cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("op_node_0", ops_choices)) + cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("op_node_1", ops_choices)) + cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("op_node_2", ops_choices)) + cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("op_node_3", ops_choices)) + cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("op_node_4", ops_choices)) + cat = [i for i in range((VERTICES * (VERTICES - 1)) // 2)] + for i in range(MAX_EDGES): + cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("edge_%d" % i, cat)) + return cs + + +class NASCifar10C(NASCifar10): + def objective_function(self, config, budget=108): + + edge_prob = [] + for i in range(VERTICES * (VERTICES - 1) // 2): + edge_prob.append(config["edge_%d" % i]) + + idx = np.argsort(edge_prob)[::-1][:config["num_edges"]] + binay_encoding = np.zeros(len(edge_prob)) + binay_encoding[idx] = 1 + matrix = np.zeros([VERTICES, VERTICES], dtype=np.int8) + idx = np.triu_indices(matrix.shape[0], k=1) + for i in range(VERTICES * (VERTICES - 1) // 2): + row = idx[0][i] + col = idx[1][i] + matrix[row, col] = binay_encoding[i] + + if graph_util.num_edges(matrix) > MAX_EDGES: + self.record_invalid(config, 1, 1, 0) + return 1, 0 + + labeling = [config["op_node_%d" % i] for i in range(5)] + labeling = ['input'] + list(labeling) + ['output'] + model_spec = api.ModelSpec(matrix, labeling) + try: + data = self.dataset.query(model_spec, epochs=budget) + except api.OutOfDomainError: + self.record_invalid(config, 1, 1, 0) + return 1, 0 + + self.record_valid(config, data, model_spec) + + return 1 - data["validation_accuracy"], data["training_time"] + + @staticmethod + def get_configuration_space(): + cs = ConfigSpace.ConfigurationSpace() + + ops_choices = ['conv1x1-bn-relu', 'conv3x3-bn-relu', 'maxpool3x3'] + cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("op_node_0", ops_choices)) + cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("op_node_1", ops_choices)) + cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("op_node_2", ops_choices)) + cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("op_node_3", ops_choices)) + cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("op_node_4", ops_choices)) + + cs.add_hyperparameter(ConfigSpace.UniformIntegerHyperparameter("num_edges", 0, MAX_EDGES)) + + for i in range(VERTICES * (VERTICES - 1) // 2): + cs.add_hyperparameter(ConfigSpace.UniformFloatHyperparameter("edge_%d" % i, 0, 1)) + return cs \ No newline at end of file diff --git a/python/ray/tune/examples/NASCifar_Trainable.py b/python/ray/tune/examples/NASCifar_Trainable.py new file mode 100644 index 000000000000..1e3e1b1ed36d --- /dev/null +++ b/python/ray/tune/examples/NASCifar_Trainable.py @@ -0,0 +1,100 @@ +""" Trainable using NASCifar Benchmark from Tabular Benchmarks for Hyperparameter Optimization and Neural Architecture Search """ +from __future__ import print_function + +import os +from ray.tune import Trainable +from NASCifar10 import NASCifar10A, NASCifar10B, NASCifar10C +import urllib.request + + +class NASCifar10ATrainable(Trainable): + def __init__(self, config=None, logger_creator=None): + # download dataset + urllib.request.urlretrieve('https://storage.googleapis.com/nasbench/nasbench_full.tfrecord', './') + + self._global_start = config.get("start", time.time()) + self._trial_start = time.time() + cwd = os.getcwd() + self.config = config + super(NASCifar10ATrainable, self).__init__(config, logger_creator) + if ray.worker._mode() == ray.worker.LOCAL_MODE: + os.chdir(cwd) + self.net = NASCifar10A('./') + self.iteration = 0 + + def _train(self): + acc, time = self.net.objective_function(self.config, self.iteration) + self.iteration += 1 + return {"validation_accuracy": acc} + + def _save(self, checkpoint_dir): + path = os.path.join(checkpoint_dir, "checkpoint") + with open(path, "w") as f: + f.write(json.dumps({"iteration": self.iteration})) + return path + + def _restore(self, checkpoint_path): + with open(checkpoint_path) as f: + self.iteration = json.loads(f.read())["iteration"] + + +class NASCifar10BTrainable(Trainable): + def __init__(self, config=None, logger_creator=None): + # download dataset + urllib.request.urlretrieve('https://storage.googleapis.com/nasbench/nasbench_full.tfrecord', './') + + self._global_start = config.get("start", time.time()) + self._trial_start = time.time() + cwd = os.getcwd() + self.config = config + super(NASCifar10BTrainable, self).__init__(config, logger_creator) + if ray.worker._mode() == ray.worker.LOCAL_MODE: + os.chdir(cwd) + self.net = NASCifar10B('./') + self.iteration = 0 + + def _train(self): + acc, time = self.net.objective_function(self.config, self.iteration) + self.iteration += 1 + return {"validation_accuracy": acc} + + def _save(self, checkpoint_dir): + path = os.path.join(checkpoint_dir, "checkpoint") + with open(path, "w") as f: + f.write(json.dumps({"iteration": self.iteration})) + return path + + def _restore(self, checkpoint_path): + with open(checkpoint_path) as f: + self.iteration = json.loads(f.read())["iteration"] + + +class NASCifar10CTrainable(Trainable): + def __init__(self, config=None, logger_creator=None): + # download dataset + urllib.request.urlretrieve('https://storage.googleapis.com/nasbench/nasbench_full.tfrecord', './') + + self._global_start = config.get("start", time.time()) + self._trial_start = time.time() + cwd = os.getcwd() + self.config = config + super(NASCifar10CTrainable, self).__init__(config, logger_creator) + if ray.worker._mode() == ray.worker.LOCAL_MODE: + os.chdir(cwd) + self.net = NASCifar10C('./') + self.iteration = 0 + + def _train(self): + acc, time = self.net.objective_function(self.config, self.iteration) + self.iteration += 1 + return {"validation_accuracy": acc} + + def _save(self, checkpoint_dir): + path = os.path.join(checkpoint_dir, "checkpoint") + with open(path, "w") as f: + f.write(json.dumps({"iteration": self.iteration})) + return path + + def _restore(self, checkpoint_path): + with open(checkpoint_path) as f: + self.iteration = json.loads(f.read())["iteration"] \ No newline at end of file From b3333a13eaf6da82851e80806ac6f7220fa26352 Mon Sep 17 00:00:00 2001 From: Lisa Dunlap Date: Mon, 5 Aug 2019 20:44:44 -0700 Subject: [PATCH 02/31] added bohb and examples --- python/ray/tune/examples/bohb_example.py | 99 +++++++++++ python/ray/tune/schedulers/bohb.py | 62 +++++++ python/ray/tune/schedulers/hb_bohb.py | 216 +++++++++++++++++++++++ 3 files changed, 377 insertions(+) create mode 100644 python/ray/tune/examples/bohb_example.py create mode 100644 python/ray/tune/schedulers/bohb.py create mode 100644 python/ray/tune/schedulers/hb_bohb.py diff --git a/python/ray/tune/examples/bohb_example.py b/python/ray/tune/examples/bohb_example.py new file mode 100644 index 000000000000..35b8c4b341c6 --- /dev/null +++ b/python/ray/tune/examples/bohb_example.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import argparse +import json +import os +import random + +import numpy as np +import ConfigSpace +import ConfigSpace.hyperparameters as CSH +import ConfigSpace.util + +import ray +from ray.tune import Trainable, run, sample_from +from ray.tune.schedulers import TuneBOHB, HyperBandForBOHB + + +class MyTrainableClass(Trainable): + """Example agent whose learning curve is a random sigmoid. + + The dummy hyperparameters "width" and "height" determine the slope and + maximum reward value reached. + """ + + def _setup(self, config): + self.timestep = 0 + + def _train(self): + self.timestep += 1 + v = np.tanh(float(self.timestep) / self.config.get("width", 1)) + v *= self.config.get("height", 1) + + # Here we use `episode_reward_mean`, but you can also report other + # objectives such as loss or accuracy. + return {"episode_reward_mean": v} + + def _save(self, checkpoint_dir): + path = os.path.join(checkpoint_dir, "checkpoint") + with open(path, "w") as f: + f.write(json.dumps({"timestep": self.timestep})) + return path + + def _restore(self, checkpoint_path): + with open(checkpoint_path) as f: + self.timestep = json.loads(f.read())["timestep"] + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--smoke-test", action="store_true", help="Finish quickly for testing") + parser.add_argument( + "--ray-redis-address", + help="Address of Ray cluster for seamless distributed execution.") + args, _ = parser.parse_known_args() + ray.init(redis_address=args.ray_redis_address) + + # asynchronous hyperband early stopping, configured with + # `episode_reward_mean` as the + # objective and `training_iteration` as the time unit, + # which is automatically filled by Tune. + + # BOHB uses ConfigSpace for their hyperparameter search space + CS = ConfigSpace + config_space = CS.ConfigurationSpace() + config_space.add_hyperparameter( + CS.UniformFloatHyperparameter('height', lower=10, upper=100)) + config_space.add_hyperparameter( + CS.UniformFloatHyperparameter('width', lower=0, upper=100)) + + bohb = HyperBandForBOHB( + time_attr="training_iteration", + metric="episode_reward_mean", + grace_period=5, + max_t=100) + + run(MyTrainableClass, + name="bohb_test", + scheduler=bohb, + search_alg=TuneBOHB( + config_space, + max_concurrent=4, + # num_concurrent=100, + reward_attr="episode_reward_mean", + ), + **{ + "stop": { + "training_iteration": 1 if args.smoke_test else 99999 + }, + "num_samples": 20, + "resources_per_trial": { + "cpu": 1, + "gpu": 0 + }, + }) diff --git a/python/ray/tune/schedulers/bohb.py b/python/ray/tune/schedulers/bohb.py new file mode 100644 index 000000000000..16d1909d97c8 --- /dev/null +++ b/python/ray/tune/schedulers/bohb.py @@ -0,0 +1,62 @@ +import ray +from ray import tune +from ray.tune.suggest import SuggestionAlgorithm + + +class TuneBOHB(SuggestionAlgorithm): + def __init__(self, + space, + max_concurrent=8, + reward_attr="neg_mean_loss", + bohb_config=None): + self._max_concurrent = max_concurrent + self.trial_to_params = {} + self.running = set() + self.paused = set() + self.reward_attr = reward_attr + bohb_config = bohb_config or {} + self.bohber = BOHB(space, **bohb_config) + super(TuneBOHB, self).__init__() + + def _suggest(self, trial_id): + if len(self.running) < self._max_concurrent: + config, info = self.bohber.get_config() + self.trial_to_params[trial_id] = list(config) + self.running.add(trial_id) + return config + return None + + def on_trial_result(self, trial_id, result): + if trial_id not in self.paused: + self.running.add(trial_id) + if "budget" in result.get("hyperband_info", {}): + hbs_wrapper = self.to_wrapper(trial_id, result) + print("adding new result", vars(hbs_wrapper)) + self.bohber.new_result(hbs_wrapper) + + def on_trial_complete(self, + trial_id, + result=None, + error=False, + early_terminated=False): + del self.trial_to_params[trial_id] + if trial_id in self.paused: + self.paused.remove(trial_id) + elif trial_id in self.running: + self.running.remove(trial_id) + else: + import ipdb; ipdb.set_trace() + + + def to_wrapper(self, trial_id, result): + return JobWrapper( + -result[self.reward_attr], result["hyperband_info"]["budget"], + {k: result["config"][k] for k in self.trial_to_params[trial_id]}) + + def on_pause(self, trial_id): + self.paused.add(trial_id) + self.running.remove(trial_id) + + def on_unpause(self, trial_id): + self.paused.remove(trial_id) + self.running.add(trial_id) \ No newline at end of file diff --git a/python/ray/tune/schedulers/hb_bohb.py b/python/ray/tune/schedulers/hb_bohb.py new file mode 100644 index 000000000000..c4052d25137a --- /dev/null +++ b/python/ray/tune/schedulers/hb_bohb.py @@ -0,0 +1,216 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import collections +import numpy as np +import logging + +from ray.tune.schedulers.trial_scheduler import FIFOScheduler, TrialScheduler +from ray.tune.schedulers.hyperband import HyperBandScheduler, Bracket +from ray.tune.trial import Trial + +logger = logging.getLogger(__name__) + + +class HyperBandForBOHB(HyperBandScheduler): + """Implements the HyperBand early stopping algorithm. + + @!!!!!!!!!!!!!! Key changes: + Bracket filling is reversed + Trial results + + + Args: + time_attr (str): The training result attr to use for comparing time. + Note that you can pass in something non-temporal such as + `training_iteration` as a measure of progress, the only requirement + is that the attribute should increase monotonically. + reward_attr (str): The training result objective value attribute. As + with `time_attr`, this may refer to any objective value. Stopping + procedures will use this attribute. + max_t (int): max time units per trial. Trials will be stopped after + max_t time units (determined by time_attr) have passed. + The scheduler will terminate trials after this time has passed. + Note that this is different from the semantics of `max_t` as + mentioned in the original HyperBand paper. + """ + + def on_trial_add(self, trial_runner, trial): + """Adds new trial. + + On a new trial add, if current bracket is not filled, + add to current bracket. Else, if current band is not filled, + create new bracket, add to current bracket. + Else, create new iteration, create new bracket, add to bracket.""" + + cur_bracket = self._state["bracket"] + cur_band = self._hyperbands[self._state["band_idx"]] + if cur_bracket is None or cur_bracket.filled(): + print("Adding a new bracket.") + # import ipdb; ipdb.set_trace() + retry = True + while retry: + # if current iteration is filled, create new iteration + if self._cur_band_filled(): + cur_band = [] + self._hyperbands.append(cur_band) + self._state["band_idx"] += 1 + + #### MAIN CHANGE HERE - largest bracket first! + # cur_band will always be less than s_max_1 or else filled + s = self._s_max_1 - len(cur_band) - 1 + assert s >= 0, "Current band is filled!" + #### MAIN CHANGE HERE! + if self._get_r0(s) == 0: + logger.info("Bracket too small - Retrying...") + cur_bracket = None + else: + retry = False + cur_bracket = ContinuationBracket(self._time_attr, self._get_n0(s), + self._get_r0(s), self._max_t_attr, + self._eta, s) + cur_band.append(cur_bracket) + self._state["bracket"] = cur_bracket + + self._state["bracket"].add_trial(trial) + self._trial_info[trial] = cur_bracket, self._state["band_idx"] + + def on_trial_result(self, trial_runner, trial, result): + """If bracket is finished, all trials will be stopped. + + If a given trial finishes and bracket iteration is not done, + the trial will be paused and resources will be given up. + + This scheduler will not start trials but will stop trials. + The current running trial will not be handled, + as the trialrunner will be given control to handle it.""" + from collections import Counter + result["hyperband_info"] = {} + bracket, _ = self._trial_info[trial] + bracket.update_trial_stats(trial, result) + + if bracket.continue_trial(trial): + return TrialScheduler.CONTINUE + + result["hyperband_info"]["budget"] = bracket._cumul_r + + #### MAIN CHANGE HERE! + statuses = [(t, t.status) for t in bracket._live_trials] + if not bracket.filled() or any( + status != Trial.PAUSED for t, status in statuses + if t is not trial): + print(Counter([status for _, status in statuses])) + trial_runner._search_alg.on_pause(trial.trial_id) + return TrialScheduler.PAUSE + # import ipdb; ipdb.set_trace() + action = self._process_bracket(trial_runner, bracket, trial) + return action + + def _process_bracket(self, trial_runner, bracket, trial): + """This is called whenever a trial makes progress. + + When all live trials in the bracket have no more iterations left, + Trials will be successively halved. If bracket is done, all + non-running trials will be stopped and cleaned up, + and during each halving phase, bad trials will be stopped while good + trials will return to "PENDING".""" + + action = TrialScheduler.PAUSE + if bracket.cur_iter_done(): + if bracket.finished(): + bracket.cleanup_full(trial_runner) + return TrialScheduler.STOP + + good, bad = bracket.successive_halving(self._reward_attr) + # kill bad trials + self._num_stopped += len(bad) + for t in bad: + if t.status == Trial.PAUSED: + trial_runner.stop_trial(t) + elif t.status == Trial.RUNNING: + bracket.cleanup_trial(t) + action = TrialScheduler.STOP + else: + raise Exception("Trial with unexpected status encountered") + + # ready the good trials - if trial is too far ahead, don't continue + for t in good: + if t.status not in [Trial.PAUSED, Trial.RUNNING]: + raise Exception("Trial with unexpected status encountered") + if bracket.continue_trial(t): + if t.status == Trial.PAUSED: + trial_runner.trial_executor.unpause_trial(t) + #### MAIN CHANGE HERE! + trial_runner._search_alg.on_unpause(t.trial_id) + #### MAIN CHANGE HERE! + elif t.status == Trial.RUNNING: + action = TrialScheduler.CONTINUE + return action + + def choose_trial_to_run(self, trial_runner): + """Fair scheduling within iteration by completion percentage. + + List of trials not used since all trials are tracked as state + of scheduler. If iteration is occupied (ie, no trials to run), + then look into next iteration. + """ + + for hyperband in self._hyperbands: + # band will have None entries if no resources + # are to be allocated to that bracket. + scrubbed = [b for b in hyperband if b is not None] + for bracket in scrubbed: + for trial in bracket.current_trials(): + if (trial.status == Trial.PENDING + and trial_runner.has_resources(trial.resources)): + return trial + #### MAIN CHANGE HERE! + if not any(t.status == Trial.RUNNING for t in trial_runner.get_trials()): + for hyperband in self._hyperbands: + for bracket in hyperband: + if bracket and any( + trial.status == Trial.PAUSED for trial in bracket.current_trials()): + self._process_bracket(trial_runner, bracket, None) + #### MAIN CHANGE HERE! + return None + +class ContinuationBracket(Bracket): + """Logical object for tracking Hyperband bracket progress. Keeps track + of proper parameters as designated by HyperBand. + + Also keeps track of progress to ensure good scheduling. + """ + + def successive_halving(self, reward_attr): + assert self._halves > 0 + self._halves -= 1 + self._n /= self._eta + self._n = int(np.ceil(self._n)) + + self._r *= self._eta + self._r = int(min(self._r, self._max_t_attr - self._cumul_r)) + + # MAIN CHANGE HERE - don't accumulate R. + self._cumul_r = self._r + ################### + sorted_trials = sorted( + self._live_trials, key=lambda t: self._live_trials[t][reward_attr]) + + good, bad = sorted_trials[-self._n:], sorted_trials[:-self._n] + return good, bad + + def update_trial_stats(self, trial, result): + """Update result for trial. Called after trial has finished + an iteration - will decrement iteration count. + + TODO(rliaw): The other alternative is to keep the trials + in and make sure they're not set as pending later.""" + + assert trial in self._live_trials + assert self._get_result_time(result) >= 0 + + delta = self._get_result_time(result) - \ + self._get_result_time(self._live_trials[trial]) + self._completed_progress += delta + self._live_trials[trial] = result From 0b418b5e42d0333f09d26698b21b2b6cf87418bb Mon Sep 17 00:00:00 2001 From: Lisa Dunlap Date: Mon, 5 Aug 2019 20:47:19 -0700 Subject: [PATCH 03/31] added bohb and example --- python/ray/tune/examples/FCNet.py | 196 --------------- python/ray/tune/examples/FCNet_Trainable.py | 157 ------------ python/ray/tune/examples/NASCifar10.py | 226 ------------------ .../ray/tune/examples/NASCifar_Trainable.py | 100 -------- 4 files changed, 679 deletions(-) delete mode 100644 python/ray/tune/examples/FCNet.py delete mode 100644 python/ray/tune/examples/FCNet_Trainable.py delete mode 100644 python/ray/tune/examples/NASCifar10.py delete mode 100644 python/ray/tune/examples/NASCifar_Trainable.py diff --git a/python/ray/tune/examples/FCNet.py b/python/ray/tune/examples/FCNet.py deleted file mode 100644 index 3d7afafd6402..000000000000 --- a/python/ray/tune/examples/FCNet.py +++ /dev/null @@ -1,196 +0,0 @@ -""" Code from https://github.com/automl/nas_benchmarks """ - -import json -import os - -import ConfigSpace -import h5py -import numpy as np - - -class FCNetBenchmark(object): - - def __init__(self, path, dataset="fcnet_protein_structure_data.hdf5", seed=None): - - cs = self.get_configuration_space() - self.names = [h.name for h in cs.get_hyperparameters()] - - self.data = h5py.File(os.path.join(path, dataset), "r") - - self.X = [] - self.y = [] - self.c = [] - - self.rng = np.random.RandomState(seed) - - def get_best_configuration(self): - - """ - Returns the best configuration in the dataset that achieves the lowest test performance. - - :return: Returns tuple with the best configuration, its final validation performance and its test performance - """ - - configs, te, ve = [], [], [] - for k in self.data.keys(): - configs.append(json.loads(k)) - te.append(np.mean(self.data[k]["final_test_error"])) - ve.append(np.mean(self.data[k]["valid_mse"][:, -1])) - - b = np.argmin(te) - - return configs[b], ve[b], te[b] - - def objective_function(self, config, budget=100, **kwargs): - - assert 0 < budget <= 100 # check whether budget is in the correct bounds - - i = self.rng.randint(4) - - if type(config) == ConfigSpace.Configuration: - k = json.dumps(config.get_dictionary(), sort_keys=True) - else: - k = json.dumps(config, sort_keys=True) - - valid = self.data[k]["valid_mse"][i] - runtime = self.data[k]["runtime"][i] - - time_per_epoch = runtime / 100 # divide by the maximum number of epochs - - rt = time_per_epoch * budget - - self.X.append(config) - self.y.append(valid[budget - 1]) - self.c.append(rt) - - return valid[budget - 1], rt - - def objective_function_learning_curve(self, config, budget=100): - - assert 0 < budget <= 100 # check whether budget is in the correct bounds - - index = self.rng.randint(4) - - if type(config) == ConfigSpace.Configuration: - k = json.dumps(config.get_dictionary(), sort_keys=True) - else: - k = json.dumps(config, sort_keys=True) - - lc = [self.data[k]["valid_mse"][index][i] for i in range(budget)] - runtime = self.data[k]["runtime"][index] - - time_per_epoch = runtime / 100 # divide by the maximum number of epochs - - rt = [time_per_epoch * (i + 1) for i in range(budget)] - - self.X.append(config) - self.y.append(lc[-1]) - self.c.append(rt[-1]) - - return lc, rt - - def objective_function_deterministic(self, config, budget=100, index=0, **kwargs): - - assert 0 < budget <= 100 # check whether budget is in the correct bounds - - if type(config) == ConfigSpace.Configuration: - k = json.dumps(config.get_dictionary(), sort_keys=True) - else: - k = json.dumps(config, sort_keys=True) - - valid = self.data[k]["valid_mse"][index] - runtime = self.data[k]["runtime"][index] - - time_per_epoch = runtime / 100 # divide by the maximum number of epochs - - rt = time_per_epoch * budget - - self.X.append(config) - self.y.append(valid[budget - 1]) - self.c.append(rt) - - return valid[budget - 1], rt - - def objective_function_test(self, config, **kwargs): - - if type(config) == ConfigSpace.Configuration: - k = json.dumps(config.get_dictionary(), sort_keys=True) - else: - k = json.dumps(config, sort_keys=True) - - test = np.mean(self.data[k]["final_test_error"]) - runtime = np.mean(self.data[k]["runtime"]) - - return test, runtime - - def get_results(self): - - inc, y_star_valid, y_star_test = self.get_best_configuration() - - regret_validation = [] - regret_test = [] - runtime = [] - rt = 0 - - inc_valid = np.inf - inc_test = np.inf - - for i in range(len(self.X)): - - if inc_valid > self.y[i]: - inc_valid = self.y[i] - inc_test, _ = self.objective_function_test(self.X[i]) - - regret_validation.append(float(inc_valid - y_star_valid)) - regret_test.append(float(inc_test - y_star_test)) - rt += self.c[i] - runtime.append(float(rt)) - - res = dict() - res['regret_validation'] = regret_validation - res['regret_test'] = regret_test - res['runtime'] = runtime - - return res - - @staticmethod - def get_configuration_space(): - cs = ConfigSpace.ConfigurationSpace() - - cs.add_hyperparameter(ConfigSpace.OrdinalHyperparameter("n_units_1", [16, 32, 64, 128, 256, 512])) - cs.add_hyperparameter(ConfigSpace.OrdinalHyperparameter("n_units_2", [16, 32, 64, 128, 256, 512])) - cs.add_hyperparameter(ConfigSpace.OrdinalHyperparameter("dropout_1", [0.0, 0.3, 0.6])) - cs.add_hyperparameter(ConfigSpace.OrdinalHyperparameter("dropout_2", [0.0, 0.3, 0.6])) - cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("activation_fn_1", ["tanh", "relu"])) - cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("activation_fn_2", ["tanh", "relu"])) - cs.add_hyperparameter( - ConfigSpace.OrdinalHyperparameter("init_lr", [5 * 1e-4, 1e-3, 5 * 1e-3, 1e-2, 5 * 1e-2, 1e-1])) - cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("lr_schedule", ["cosine", "const"])) - cs.add_hyperparameter(ConfigSpace.OrdinalHyperparameter("batch_size", [8, 16, 32, 64])) - return cs - - -class FCNetSliceLocalizationBenchmark(FCNetBenchmark): - - def __init__(self, data_dir="./"): - super(FCNetSliceLocalizationBenchmark, self).__init__(path=data_dir, - dataset="fcnet_slice_localization_data.hdf5") - - -class FCNetProteinStructureBenchmark(FCNetBenchmark): - - def __init__(self, data_dir="./"): - super(FCNetProteinStructureBenchmark, self).__init__(path=data_dir, dataset="fcnet_protein_structure_data.hdf5") - - -class FCNetNavalPropulsionBenchmark(FCNetBenchmark): - - def __init__(self, data_dir="./"): - super(FCNetNavalPropulsionBenchmark, self).__init__(path=data_dir, dataset="fcnet_naval_propulsion_data.hdf5") - - -class FCNetParkinsonsTelemonitoringBenchmark(FCNetBenchmark): - - def __init__(self, data_dir="./"): - super(FCNetParkinsonsTelemonitoringBenchmark, self).__init__(path=data_dir, - dataset="fcnet_parkinsons_telemonitoring_data.hdf5") \ No newline at end of file diff --git a/python/ray/tune/examples/FCNet_Trainable.py b/python/ray/tune/examples/FCNet_Trainable.py deleted file mode 100644 index 6372f1d59aba..000000000000 --- a/python/ray/tune/examples/FCNet_Trainable.py +++ /dev/null @@ -1,157 +0,0 @@ -""" Trainable using FCNet Benchmark from Tabular Benchmarks for Hyperparameter Optimization and Neural Architecture Search """ -from __future__ import print_function - -import os - -from ray.tune import Trainable -import tarfile -import urllib -from FCNet import FCNetBenchmark, FCNetProteinStructureBenchmark, FCNetSliceLocalizationBenchmark, FCNetNavalPropulsionBenchmark, FCNetParkinsonsTelemonitoringBenchmark - - -class FCNetProteinStructureTrainable(Trainable): - def __init__(self, config=None, logger_creator=None): - - # download dataset - file_tmp = urllib.urlretrieve('http://ml4aad.org/wp-content/uploads/2019/01/fcnet_tabular_benchmarks.tar.gz', filename=None)[0] - base_name = os.path.basename('http://ml4aad.org/wp-content/uploads/2019/01/fcnet_tabular_benchmarks.tar.gz') - - file_name, file_extension = os.path.splitext(base_name) - tar = tarfile.open(file_tmp) - tar.extractall(file_name) - - self._global_start = config.get("start", time.time()) - self._trial_start = time.time() - self.config = config - cwd = os.getcwd() - super(FCNetProteinStructureTrainable, self).__init__(config, logger_creator) - if ray.worker._mode() == ray.worker.LOCAL_MODE: - os.chdir(cwd) - self.net =FCNetProteinStructureBenchmark() - self.iteration = 0 - - def _train(self): - acc, time = self.net.objective_function(self.config, self.iteration) - self.iteration += 1 - return {"validation_accuracy": acc, 'runtime': time} - - def _save(self, checkpoint_dir): - path = os.path.join(checkpoint_dir, "checkpoint") - with open(path, "w") as f: - f.write(json.dumps({"iteration": self.iteration})) - return path - - def _restore(self, checkpoint_path): - with open(checkpoint_path) as f: - self.iteration = json.loads(f.read())["iteration"] - - -class FCNetSliceLocalizationTrainable(Trainable): - def __init__(self, config=None, logger_creator=None): - - # download dataset - file_tmp = urllib.urlretrieve('http://ml4aad.org/wp-content/uploads/2019/01/fcnet_tabular_benchmarks.tar.gz', filename=None)[0] - base_name = os.path.basename('http://ml4aad.org/wp-content/uploads/2019/01/fcnet_tabular_benchmarks.tar.gz') - - file_name, file_extension = os.path.splitext(base_name) - tar = tarfile.open(file_tmp) - tar.extractall(file_name) - - self._global_start = config.get("start", time.time()) - self._trial_start = time.time() - self.config = config - cwd = os.getcwd() - super(FCNetSliceLocalizationTrainable, self).__init__(config, logger_creator) - if ray.worker._mode() == ray.worker.LOCAL_MODE: - os.chdir(cwd) - self.net =FCNetProteinStructureBenchmark() - self.iteration = 0 - - def _train(self): - acc, time = self.net.objective_function(self.config, self.iteration) - self.iteration += 1 - return {"validation_accuracy": acc} - - def _save(self, checkpoint_dir): - path = os.path.join(checkpoint_dir, "checkpoint") - with open(path, "w") as f: - f.write(json.dumps({"iteration": self.iteration})) - return path - - def _restore(self, checkpoint_path): - with open(checkpoint_path) as f: - self.iteration = json.loads(f.read())["iteration"] - - -class FCNetNavalPropulsionTrainable(Trainable): - def __init__(self, config=None, logger_creator=None): - - # download dataset - file_tmp = urllib.urlretrieve('http://ml4aad.org/wp-content/uploads/2019/01/fcnet_tabular_benchmarks.tar.gz', filename=None)[0] - base_name = os.path.basename('http://ml4aad.org/wp-content/uploads/2019/01/fcnet_tabular_benchmarks.tar.gz') - - file_name, file_extension = os.path.splitext(base_name) - tar = tarfile.open(file_tmp) - tar.extractall(file_name) - - self._global_start = config.get("start", time.time()) - self._trial_start = time.time() - self.config = config - cwd = os.getcwd() - super(FCNetNavalPropulsionTrainable, self).__init__(config, logger_creator) - if ray.worker._mode() == ray.worker.LOCAL_MODE: - os.chdir(cwd) - self.net =FCNetProteinStructureBenchmark() - self.iteration = 0 - - def _train(self): - acc, time = self.net.objective_function(self.config, self.iteration) - self.iteration += 1 - return {"validation_accuracy": acc} - - def _save(self, checkpoint_dir): - path = os.path.join(checkpoint_dir, "checkpoint") - with open(path, "w") as f: - f.write(json.dumps({"iteration": self.iteration})) - return path - - def _restore(self, checkpoint_path): - with open(checkpoint_path) as f: - self.iteration = json.loads(f.read())["iteration"] - - -class FCNetParkinsonsTelemonitoringTrainable(Trainable): - def __init__(self, config=None, logger_creator=None): - - # download dataset - file_tmp = urllib.urlretrieve('http://ml4aad.org/wp-content/uploads/2019/01/fcnet_tabular_benchmarks.tar.gz', filename=None)[0] - base_name = os.path.basename('http://ml4aad.org/wp-content/uploads/2019/01/fcnet_tabular_benchmarks.tar.gz') - - file_name, file_extension = os.path.splitext(base_name) - tar = tarfile.open(file_tmp) - tar.extractall(file_name) - - self._global_start = config.get("start", time.time()) - self._trial_start = time.time() - self.config = config - cwd = os.getcwd() - super(FCNetParkinsonsTelemonitoringTrainable, self).__init__(config, logger_creator) - if ray.worker._mode() == ray.worker.LOCAL_MODE: - os.chdir(cwd) - self.net =FCNetProteinStructureBenchmark() - self.iteration = 0 - - def _train(self): - acc, time = self.net.objective_function(self.config, self.iteration) - self.iteration += 1 - return {"validation_accuracy": acc} - - def _save(self, checkpoint_dir): - path = os.path.join(checkpoint_dir, "checkpoint") - with open(path, "w") as f: - f.write(json.dumps({"iteration": self.iteration})) - return path - - def _restore(self, checkpoint_path): - with open(checkpoint_path) as f: - self.iteration = json.loads(f.read())["iteration"] \ No newline at end of file diff --git a/python/ray/tune/examples/NASCifar10.py b/python/ray/tune/examples/NASCifar10.py deleted file mode 100644 index 6ee3b32a3bf4..000000000000 --- a/python/ray/tune/examples/NASCifar10.py +++ /dev/null @@ -1,226 +0,0 @@ -""" Code from https://github.com/automl/nas_benchmarks """ - -import os - -import ConfigSpace -import numpy as np -from nasbench import api -from nasbench.lib import graph_util - -MAX_EDGES = 9 -VERTICES = 7 - - -class NASCifar10(object): - - def __init__(self, data_dir): - - self.dataset = api.NASBench(os.path.join(data_dir, 'nasbench_full.tfrecord')) - self.X = [] - self.y_valid = [] - self.y_test = [] - self.costs = [] - - self.y_star_valid = 0.04944576819737756 # lowest mean validation error - self.y_star_test = 0.056824247042338016 # lowest mean test error - - @staticmethod - def objective_function(self, config): - pass - - def record_invalid(self, config, valid, test, costs): - self.X.append(config) - self.y_valid.append(valid) - self.y_test.append(test) - self.costs.append(costs) - - def record_valid(self, config, data, model_spec): - - self.X.append(config) - - # compute mean test error for the final budget - _, metrics = self.dataset.get_metrics_from_spec(model_spec) - mean_test_error = 1 - np.mean([metrics[108][i]["final_test_accuracy"] for i in range(3)]) - self.y_test.append(mean_test_error) - - # compute validation error for the chosen budget - valid_error = 1 - data["validation_accuracy"] - self.y_valid.append(valid_error) - - runtime = data["training_time"] - self.costs.append(runtime) - - @staticmethod - def get_configuration_space(): - pass - - def get_results(self, ignore_invalid_configs=False): - - regret_validation = [] - regret_test = [] - runtime = [] - rt = 0 - - inc_valid = np.inf - inc_test = np.inf - - for i in range(len(self.X)): - - if ignore_invalid_configs and self.costs[i] == 0: - continue - - if inc_valid > self.y_valid[i]: - inc_valid = self.y_valid[i] - inc_test = self.y_test[i] - - regret_validation.append(float(inc_valid - self.y_star_valid)) - regret_test.append(float(inc_test - self.y_star_test)) - rt += self.costs[i] - runtime.append(float(rt)) - - res = dict() - res['regret_validation'] = regret_validation - res['regret_test'] = regret_test - res['runtime'] = runtime - - return res - - -class NASCifar10A(NASCifar10): - def objective_function(self, config, budget=108): - matrix = np.zeros([VERTICES, VERTICES], dtype=np.int8) - idx = np.triu_indices(matrix.shape[0], k=1) - for i in range(VERTICES * (VERTICES - 1) // 2): - row = idx[0][i] - col = idx[1][i] - matrix[row, col] = config["edge_%d" % i] - - # if not graph_util.is_full_dag(matrix) or graph_util.num_edges(matrix) > MAX_EDGES: - if graph_util.num_edges(matrix) > MAX_EDGES: - self.record_invalid(config, 1, 1, 0) - return 1, 0 - - labeling = [config["op_node_%d" % i] for i in range(5)] - labeling = ['input'] + list(labeling) + ['output'] - model_spec = api.ModelSpec(matrix, labeling) - try: - data = self.dataset.query(model_spec, epochs=budget) - except api.OutOfDomainError: - self.record_invalid(config, 1, 1, 0) - return 1, 0 - - self.record_valid(config, data, model_spec) - return 1 - data["validation_accuracy"], data["training_time"] - - @staticmethod - def get_configuration_space(): - cs = ConfigSpace.ConfigurationSpace() - - ops_choices = ['conv1x1-bn-relu', 'conv3x3-bn-relu', 'maxpool3x3'] - cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("op_node_0", ops_choices)) - cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("op_node_1", ops_choices)) - cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("op_node_2", ops_choices)) - cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("op_node_3", ops_choices)) - cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("op_node_4", ops_choices)) - for i in range(VERTICES * (VERTICES - 1) // 2): - cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("edge_%d" % i, [0, 1])) - return cs - - -class NASCifar10B(NASCifar10): - def objective_function(self, config, budget=108): - - bitlist = [0] * (VERTICES * (VERTICES - 1) // 2) - for i in range(MAX_EDGES): - bitlist[config["edge_%d" % i]] = 1 - out = 0 - for bit in bitlist: - out = (out << 1) | bit - - matrix = np.fromfunction(graph_util.gen_is_edge_fn(out), - (VERTICES, VERTICES), - dtype=np.int8) - # if not graph_util.is_full_dag(matrix) or graph_util.num_edges(matrix) > MAX_EDGES: - if graph_util.num_edges(matrix) > MAX_EDGES: - self.record_invalid(config, 1, 1, 0) - return 1, 0 - - labeling = [config["op_node_%d" % i] for i in range(5)] - labeling = ['input'] + list(labeling) + ['output'] - model_spec = api.ModelSpec(matrix, labeling) - try: - data = self.dataset.query(model_spec, epochs=budget) - except api.OutOfDomainError: - self.record_invalid(config, 1, 1, 0) - return 1, 0 - - self.record_valid(config, data, model_spec) - - return 1 - data["validation_accuracy"], data["training_time"] - - @staticmethod - def get_configuration_space(): - cs = ConfigSpace.ConfigurationSpace() - - ops_choices = ['conv1x1-bn-relu', 'conv3x3-bn-relu', 'maxpool3x3'] - cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("op_node_0", ops_choices)) - cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("op_node_1", ops_choices)) - cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("op_node_2", ops_choices)) - cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("op_node_3", ops_choices)) - cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("op_node_4", ops_choices)) - cat = [i for i in range((VERTICES * (VERTICES - 1)) // 2)] - for i in range(MAX_EDGES): - cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("edge_%d" % i, cat)) - return cs - - -class NASCifar10C(NASCifar10): - def objective_function(self, config, budget=108): - - edge_prob = [] - for i in range(VERTICES * (VERTICES - 1) // 2): - edge_prob.append(config["edge_%d" % i]) - - idx = np.argsort(edge_prob)[::-1][:config["num_edges"]] - binay_encoding = np.zeros(len(edge_prob)) - binay_encoding[idx] = 1 - matrix = np.zeros([VERTICES, VERTICES], dtype=np.int8) - idx = np.triu_indices(matrix.shape[0], k=1) - for i in range(VERTICES * (VERTICES - 1) // 2): - row = idx[0][i] - col = idx[1][i] - matrix[row, col] = binay_encoding[i] - - if graph_util.num_edges(matrix) > MAX_EDGES: - self.record_invalid(config, 1, 1, 0) - return 1, 0 - - labeling = [config["op_node_%d" % i] for i in range(5)] - labeling = ['input'] + list(labeling) + ['output'] - model_spec = api.ModelSpec(matrix, labeling) - try: - data = self.dataset.query(model_spec, epochs=budget) - except api.OutOfDomainError: - self.record_invalid(config, 1, 1, 0) - return 1, 0 - - self.record_valid(config, data, model_spec) - - return 1 - data["validation_accuracy"], data["training_time"] - - @staticmethod - def get_configuration_space(): - cs = ConfigSpace.ConfigurationSpace() - - ops_choices = ['conv1x1-bn-relu', 'conv3x3-bn-relu', 'maxpool3x3'] - cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("op_node_0", ops_choices)) - cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("op_node_1", ops_choices)) - cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("op_node_2", ops_choices)) - cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("op_node_3", ops_choices)) - cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("op_node_4", ops_choices)) - - cs.add_hyperparameter(ConfigSpace.UniformIntegerHyperparameter("num_edges", 0, MAX_EDGES)) - - for i in range(VERTICES * (VERTICES - 1) // 2): - cs.add_hyperparameter(ConfigSpace.UniformFloatHyperparameter("edge_%d" % i, 0, 1)) - return cs \ No newline at end of file diff --git a/python/ray/tune/examples/NASCifar_Trainable.py b/python/ray/tune/examples/NASCifar_Trainable.py deleted file mode 100644 index 1e3e1b1ed36d..000000000000 --- a/python/ray/tune/examples/NASCifar_Trainable.py +++ /dev/null @@ -1,100 +0,0 @@ -""" Trainable using NASCifar Benchmark from Tabular Benchmarks for Hyperparameter Optimization and Neural Architecture Search """ -from __future__ import print_function - -import os -from ray.tune import Trainable -from NASCifar10 import NASCifar10A, NASCifar10B, NASCifar10C -import urllib.request - - -class NASCifar10ATrainable(Trainable): - def __init__(self, config=None, logger_creator=None): - # download dataset - urllib.request.urlretrieve('https://storage.googleapis.com/nasbench/nasbench_full.tfrecord', './') - - self._global_start = config.get("start", time.time()) - self._trial_start = time.time() - cwd = os.getcwd() - self.config = config - super(NASCifar10ATrainable, self).__init__(config, logger_creator) - if ray.worker._mode() == ray.worker.LOCAL_MODE: - os.chdir(cwd) - self.net = NASCifar10A('./') - self.iteration = 0 - - def _train(self): - acc, time = self.net.objective_function(self.config, self.iteration) - self.iteration += 1 - return {"validation_accuracy": acc} - - def _save(self, checkpoint_dir): - path = os.path.join(checkpoint_dir, "checkpoint") - with open(path, "w") as f: - f.write(json.dumps({"iteration": self.iteration})) - return path - - def _restore(self, checkpoint_path): - with open(checkpoint_path) as f: - self.iteration = json.loads(f.read())["iteration"] - - -class NASCifar10BTrainable(Trainable): - def __init__(self, config=None, logger_creator=None): - # download dataset - urllib.request.urlretrieve('https://storage.googleapis.com/nasbench/nasbench_full.tfrecord', './') - - self._global_start = config.get("start", time.time()) - self._trial_start = time.time() - cwd = os.getcwd() - self.config = config - super(NASCifar10BTrainable, self).__init__(config, logger_creator) - if ray.worker._mode() == ray.worker.LOCAL_MODE: - os.chdir(cwd) - self.net = NASCifar10B('./') - self.iteration = 0 - - def _train(self): - acc, time = self.net.objective_function(self.config, self.iteration) - self.iteration += 1 - return {"validation_accuracy": acc} - - def _save(self, checkpoint_dir): - path = os.path.join(checkpoint_dir, "checkpoint") - with open(path, "w") as f: - f.write(json.dumps({"iteration": self.iteration})) - return path - - def _restore(self, checkpoint_path): - with open(checkpoint_path) as f: - self.iteration = json.loads(f.read())["iteration"] - - -class NASCifar10CTrainable(Trainable): - def __init__(self, config=None, logger_creator=None): - # download dataset - urllib.request.urlretrieve('https://storage.googleapis.com/nasbench/nasbench_full.tfrecord', './') - - self._global_start = config.get("start", time.time()) - self._trial_start = time.time() - cwd = os.getcwd() - self.config = config - super(NASCifar10CTrainable, self).__init__(config, logger_creator) - if ray.worker._mode() == ray.worker.LOCAL_MODE: - os.chdir(cwd) - self.net = NASCifar10C('./') - self.iteration = 0 - - def _train(self): - acc, time = self.net.objective_function(self.config, self.iteration) - self.iteration += 1 - return {"validation_accuracy": acc} - - def _save(self, checkpoint_dir): - path = os.path.join(checkpoint_dir, "checkpoint") - with open(path, "w") as f: - f.write(json.dumps({"iteration": self.iteration})) - return path - - def _restore(self, checkpoint_path): - with open(checkpoint_path) as f: - self.iteration = json.loads(f.read())["iteration"] \ No newline at end of file From 5ee0ee1654609ddcdda1064b29f7e900dde95c85 Mon Sep 17 00:00:00 2001 From: Lisa Dunlap Date: Thu, 8 Aug 2019 23:46:11 -0700 Subject: [PATCH 04/31] fixed PR changes --- python/__init__.py | 0 python/ray/tune/examples/bohb_example.py | 5 +++-- python/ray/tune/{schedulers => suggest}/bohb.py | 15 ++++++++++----- 3 files changed, 13 insertions(+), 7 deletions(-) create mode 100644 python/__init__.py rename python/ray/tune/{schedulers => suggest}/bohb.py (85%) diff --git a/python/__init__.py b/python/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/ray/tune/examples/bohb_example.py b/python/ray/tune/examples/bohb_example.py index 35b8c4b341c6..a37dc0ed1320 100644 --- a/python/ray/tune/examples/bohb_example.py +++ b/python/ray/tune/examples/bohb_example.py @@ -16,7 +16,8 @@ import ray from ray.tune import Trainable, run, sample_from -from ray.tune.schedulers import TuneBOHB, HyperBandForBOHB +from ray.tune.schedulers import HyperBandForBOHB +from ray.tune.suggest import TuneBOHB class MyTrainableClass(Trainable): @@ -85,7 +86,7 @@ def _restore(self, checkpoint_path): config_space, max_concurrent=4, # num_concurrent=100, - reward_attr="episode_reward_mean", + metric="episode_reward_mean", ), **{ "stop": { diff --git a/python/ray/tune/schedulers/bohb.py b/python/ray/tune/suggest/bohb.py similarity index 85% rename from python/ray/tune/schedulers/bohb.py rename to python/ray/tune/suggest/bohb.py index 16d1909d97c8..3f108bb5d2f6 100644 --- a/python/ray/tune/schedulers/bohb.py +++ b/python/ray/tune/suggest/bohb.py @@ -1,5 +1,10 @@ -import ray -from ray import tune +"""BOHB (Bayesian Optimization with HyperBand)""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from hpbandster import BOHB from ray.tune.suggest import SuggestionAlgorithm @@ -7,13 +12,13 @@ class TuneBOHB(SuggestionAlgorithm): def __init__(self, space, max_concurrent=8, - reward_attr="neg_mean_loss", + metric="neg_mean_loss", bohb_config=None): self._max_concurrent = max_concurrent self.trial_to_params = {} self.running = set() self.paused = set() - self.reward_attr = reward_attr + self.metric = metric bohb_config = bohb_config or {} self.bohber = BOHB(space, **bohb_config) super(TuneBOHB, self).__init__() @@ -50,7 +55,7 @@ def on_trial_complete(self, def to_wrapper(self, trial_id, result): return JobWrapper( - -result[self.reward_attr], result["hyperband_info"]["budget"], + -result[self.metric], result["hyperband_info"]["budget"], {k: result["config"][k] for k in self.trial_to_params[trial_id]}) def on_pause(self, trial_id): From d6d715b82a33c47b1386cb4dc03dd90bb0e90613 Mon Sep 17 00:00:00 2001 From: Lisa Dunlap Date: Fri, 9 Aug 2019 00:33:53 -0700 Subject: [PATCH 05/31] added reformatted bohb and docs --- doc/source/tune-searchalg.rst | 25 +++ python/ray/tune/suggest/bohb.py | 386 +++++++++++++++++++++++++++++++- 2 files changed, 410 insertions(+), 1 deletion(-) diff --git a/doc/source/tune-searchalg.rst b/doc/source/tune-searchalg.rst index 2dae8eaf4abe..d249d27b415a 100644 --- a/doc/source/tune-searchalg.rst +++ b/doc/source/tune-searchalg.rst @@ -18,6 +18,7 @@ Currently, Tune offers the following search algorithms (and library integrations - `Nevergrad `__ - `Scikit-Optimize `__ - `Ax `__ +- `BOHB `__ Variant Generation (Grid Search/Random Search) @@ -181,6 +182,30 @@ An example of this can be found in `ax_example.py `__ to perform sequential model-based hyperparameter optimization in conjunction with Hyperband. Note that this class does not extend ``ray.tune.suggest.BasicVariantGenerator``, so you will not be able to use Tune's default variant generation/search space declaration when using BOHB. Also note that BOHB is intended to be paried with the Hyperband scheduler `hp_bohb.py `__. + +In order to use this search algorithm, you will need to install HpBandSter and ConfigSpace via the following command: + +.. code-block:: bash + + $ pip install hpbandster + $ pip install ConfigSpace + +This algorithm requires using the `ConfigSpace search space specification `__. You can use TuneBOHB like follows: + +.. code-block:: python + + tune.run(... , search_alg=TuneBOHB(config_space, ... )) + +An example of this can be found in `bohb_example.py `__. + +.. autoclass:: ray.tune.suggest.bohb.TuneBOHB + :show-inheritance: + :noindex: + Contributing a New Algorithm ---------------------------- diff --git a/python/ray/tune/suggest/bohb.py b/python/ray/tune/suggest/bohb.py index 3f108bb5d2f6..fa3b2093b707 100644 --- a/python/ray/tune/suggest/bohb.py +++ b/python/ray/tune/suggest/bohb.py @@ -4,11 +4,395 @@ from __future__ import division from __future__ import print_function -from hpbandster import BOHB +import ConfigSpace +import ConfigSpace.hyperparameters as CSH +import ConfigSpace.util +import numpy as np +import scipy.stats as sps +import statsmodels.api as sm + +from hpbandster.core.base_config_generator import base_config_generator from ray.tune.suggest import SuggestionAlgorithm +class BOHB(base_config_generator): + def __init__(self, + configspace, + min_points_in_model=None, + top_n_percent=15, + num_samples=64, + random_fraction=1 / 3, + bandwidth_factor=3, + min_bandwidth=1e-3, + **kwargs): + """ + Fits for each given budget a kernel density estimator on the best N percent of the + evaluated configurations on this budget. + + + Parameters: + ----------- + configspace: ConfigSpace + Configuration space object + top_n_percent: int + Determines the percentile of configurations that will be used as training data + for the kernel density estimator, e.g if set to 10 the 10% best configurations will be considered + for training. + min_points_in_model: int + minimum number of datapoints needed to fit a model + num_samples: int + number of samples drawn to optimize EI via sampling + random_fraction: float + fraction of random configurations returned + bandwidth_factor: float + widens the bandwidth for contiuous parameters for proposed points to optimize EI + min_bandwidth: float + to keep diversity, even when all (good) samples have the same value for one of the parameters, + a minimum bandwidth (Default: 1e-3) is used instead of zero. + + """ + super().__init__(**kwargs) + self.top_n_percent = top_n_percent + self.configspace = configspace + self.bw_factor = bandwidth_factor + self.min_bandwidth = min_bandwidth + + self.min_points_in_model = min_points_in_model + if min_points_in_model is None: + self.min_points_in_model = len( + self.configspace.get_hyperparameters()) + 1 + + if self.min_points_in_model < len( + self.configspace.get_hyperparameters()) + 1: + self.logger.warning( + 'Invalid min_points_in_model value. Setting it to %i' % + (len(self.configspace.get_hyperparameters()) + 1)) + self.min_points_in_model = len( + self.configspace.get_hyperparameters()) + 1 + + self.num_samples = num_samples + self.random_fraction = random_fraction + + hps = self.configspace.get_hyperparameters() + + self.kde_vartypes = "" + self.vartypes = [] + + for h in hps: + if hasattr(h, 'sequence'): + raise RuntimeError( + 'This version on BOHB does not support ordinal hyperparameters. Please encode %s as an integer parameter!' + % (h.name)) + + if hasattr(h, 'choices'): + self.kde_vartypes += 'u' + self.vartypes += [len(h.choices)] + else: + self.kde_vartypes += 'c' + self.vartypes += [0] + + self.vartypes = np.array(self.vartypes, dtype=int) + + # store precomputed probs for the categorical parameters + self.cat_probs = [] + + self.configs = dict() + self.losses = dict() + self.good_config_rankings = dict() + self.kde_models = dict() + + def largest_budget_with_model(self): + if len(self.kde_models) == 0: + return (-float('inf')) + return (max(self.kde_models.keys())) + + def get_config(self, *args): + """ + Function to sample a new configuration + + This function is called inside Hyperband to query a new configuration + + + returns: config + should return a valid configuration + + """ + + self.logger.debug('start sampling a new configuration.') + + sample = None + info_dict = {} + + # If no model is available, sample from prior + # also mix in a fraction of random configs + if len(self.kde_models.keys() + ) == 0 or np.random.rand() < self.random_fraction: + sample = self.configspace.sample_configuration() + info_dict['model_based_pick'] = False + + best = np.inf + best_vector = None + + if sample is None: + try: + + #sample from largest budget + budget = max(self.kde_models.keys()) + + l = self.kde_models[budget]['good'].pdf + g = self.kde_models[budget]['bad'].pdf + + minimize_me = lambda x: max(1e-32, g(x)) / max(l(x), 1e-32) + + kde_good = self.kde_models[budget]['good'] + kde_bad = self.kde_models[budget]['bad'] + + for i in range(self.num_samples): + idx = np.random.randint(0, len(kde_good.data)) + datum = kde_good.data[idx] + vector = [] + + for m, bw, t in zip(datum, kde_good.bw, self.vartypes): + + bw = max(bw, self.min_bandwidth) + if t == 0: + bw = self.bw_factor * bw + try: + vector.append( + sps.truncnorm.rvs( + -m / bw, (1 - m) / bw, loc=m, + scale=bw)) + except: + self.logger.warning( + "Truncated Normal failed for:\ndatum=%s\nbandwidth=%s\nfor entry with value %s" + % (datum, kde_good.bw, m)) + self.logger.warning( + "data in the KDE:\n%s" % kde_good.data) + else: + + if np.random.rand() < (1 - bw): + vector.append(int(m)) + else: + vector.append(np.random.randint(t)) + val = minimize_me(vector) + + if not np.isfinite(val): + self.logger.warning( + 'sampled vector: %s has EI value %s' % (vector, + val)) + self.logger.warning("data in the KDEs:\n%s\n%s" % + (kde_good.data, kde_bad.data)) + self.logger.warning("bandwidth of the KDEs:\n%s\n%s" % + (kde_good.bw, kde_bad.bw)) + self.logger.warning("l(x) = %s" % (l(vector))) + self.logger.warning("g(x) = %s" % (g(vector))) + + # right now, this happens because a KDE does not contain all values for a categorical parameter + # this cannot be fixed with the statsmodels KDE, so for now, we are just going to evaluate this one + # if the good_kde has a finite value, i.e. there is no config with that value in the bad kde, so it shouldn't be terrible. + if np.isfinite(l(vector)): + best_vector = vector + break + + if val < best: + best = val + best_vector = vector + + if best_vector is None: + self.logger.debug( + "Sampling based optimization with %i samples failed -> using random configuration" + % self.num_samples) + sample = self.configspace.sample_configuration( + ).get_dictionary() + info_dict['model_based_pick'] = False + else: + self.logger.debug('best_vector: {}, {}, {}, {}'.format( + best_vector, best, l(best_vector), g(best_vector))) + for i, hp_value in enumerate(best_vector): + if isinstance( + self.configspace.get_hyperparameter( + self.configspace.get_hyperparameter_by_idx( + i)), CSH.CategoricalHyperparameter): + best_vector[i] = int(np.rint(best_vector[i])) + sample = ConfigSpace.Configuration( + self.configspace, vector=best_vector).get_dictionary() + + try: + sample = ConfigSpace.util.deactivate_inactive_hyperparameters( + configuration_space=self.configspace, + configuration=sample) + info_dict['model_based_pick'] = True + + except Exception as e: + self.logger.warning(("="*50 + "\n")*3 +\ + "Error converting configuration:\n%s"%sample+\ + "\n here is a traceback:" +\ + traceback.format_exc()) + raise (e) + + except: + self.logger.warning( + "Sampling based optimization with %i samples failed\n %s \nUsing random configuration" + % (self.num_samples, traceback.format_exc())) + sample = self.configspace.sample_configuration() + info_dict['model_based_pick'] = False + + try: + sample = ConfigSpace.util.deactivate_inactive_hyperparameters( + configuration_space=self.configspace, + configuration=sample.get_dictionary()).get_dictionary() + except Exception as e: + self.logger.warning( + "Error (%s) converting configuration: %s -> " + "using random configuration!", e, sample) + sample = self.configspace.sample_configuration().get_dictionary() + self.logger.debug('done sampling a new configuration.') + return sample, info_dict + + def impute_conditional_data(self, array): + + return_array = np.empty_like(array) + + for i in range(array.shape[0]): + datum = np.copy(array[i]) + nan_indices = np.argwhere(np.isnan(datum)).flatten() + + while (np.any(nan_indices)): + nan_idx = nan_indices[0] + valid_indices = np.argwhere(np.isfinite( + array[:, nan_idx])).flatten() + + if len(valid_indices) > 0: + # pick one of them at random and overwrite all NaN values + row_idx = np.random.choice(valid_indices) + datum[nan_indices] = array[row_idx, nan_indices] + + else: + # no good point in the data has this value activated, so fill it with a valid but random value + t = self.vartypes[nan_idx] + if t == 0: + datum[nan_idx] = np.random.rand() + else: + datum[nan_idx] = np.random.randint(t) + + nan_indices = np.argwhere(np.isnan(datum)).flatten() + return_array[i, :] = datum + return (return_array) + + def new_result(self, job, update_model=True): + """ + function to register finished runs + + Every time a run has finished, this function should be called + to register it with the result logger. If overwritten, make + sure to call this method from the base class to ensure proper + logging. + + + Parameters: + ----------- + job: hpbandster.distributed.dispatcher.Job object + contains all the info about the run + """ + + super().new_result(job) + + if job.result is None: + # One could skip crashed results, but we decided to + # assign a +inf loss and count them as bad configurations + loss = np.inf + else: + # same for non numeric losses. + # Note that this means losses of minus infinity will count as bad! + loss = job.result["loss"] if np.isfinite( + job.result["loss"]) else np.inf + + budget = job.kwargs["budget"] + + if budget not in self.configs.keys(): + self.configs[budget] = [] + self.losses[budget] = [] + + # skip model building if we already have a bigger model + if max(list(self.kde_models.keys()) + [-np.inf]) > budget: + return + + # We want to get a numerical representation of the configuration in the original space + + conf = ConfigSpace.Configuration(self.configspace, + job.kwargs["config"]) + self.configs[budget].append(conf.get_array()) + self.losses[budget].append(loss) + + # skip model building: + # a) if not enough points are available + if len(self.configs[budget]) <= self.min_points_in_model - 1: + self.logger.debug( + "Only %i run(s) for budget %f available, need more than %s -> can't build model!" + % (len(self.configs[budget]), budget, + self.min_points_in_model + 1)) + return + + # b) during warnm starting when we feed previous results in and only update once + if not update_model: + return + + train_configs = np.array(self.configs[budget]) + train_losses = np.array(self.losses[budget]) + + n_good = max(self.min_points_in_model, + (self.top_n_percent * train_configs.shape[0]) // 100) + #n_bad = min(max(self.min_points_in_model, ((100-self.top_n_percent)*train_configs.shape[0])//100), 10) + n_bad = max( + self.min_points_in_model, + ((100 - self.top_n_percent) * train_configs.shape[0]) // 100) + + # Refit KDE for the current budget + idx = np.argsort(train_losses) + + train_data_good = self.impute_conditional_data( + train_configs[idx[:n_good]]) + train_data_bad = self.impute_conditional_data( + train_configs[idx[n_good:n_good + n_bad]]) + + if train_data_good.shape[0] <= train_data_good.shape[1]: + return + if train_data_bad.shape[0] <= train_data_bad.shape[1]: + return + + #more expensive crossvalidation method + #bw_estimation = 'cv_ls' + + # quick rule of thumb + bw_estimation = 'normal_reference' + + bad_kde = sm.nonparametric.KDEMultivariate( + data=train_data_bad, var_type=self.kde_vartypes, bw=bw_estimation) + good_kde = sm.nonparametric.KDEMultivariate( + data=train_data_good, var_type=self.kde_vartypes, bw=bw_estimation) + + bad_kde.bw = np.clip(bad_kde.bw, self.min_bandwidth, None) + good_kde.bw = np.clip(good_kde.bw, self.min_bandwidth, None) + + self.kde_models[budget] = {'good': good_kde, 'bad': bad_kde} + + # update probs for the categorical parameters for later sampling + self.logger.debug( + 'done building a new model for budget %f based on %i/%i split\nBest loss for this budget:%f\n\n\n\n\n' + % (budget, n_good, n_bad, np.min(train_losses))) + + +class JobWrapper(): + """Dummy class""" + + def __init__(self, loss, budget, config): + self.result = {"loss": loss} + self.kwargs = {"budget": budget, "config": config.copy()} + self.exception = None + class TuneBOHB(SuggestionAlgorithm): + """Tune Suggestion Algorithm for BOHB code""" + def __init__(self, space, max_concurrent=8, From d69f880790689837eeafd4f27735b1da29dfed5c Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Fri, 9 Aug 2019 01:04:27 -0700 Subject: [PATCH 06/31] Delete __init__.py --- python/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 python/__init__.py diff --git a/python/__init__.py b/python/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 From 00edd953ba6a353666cd288a4abdc1175d87a26d Mon Sep 17 00:00:00 2001 From: Lisa Dunlap Date: Fri, 9 Aug 2019 01:08:33 -0700 Subject: [PATCH 07/31] removed BOHB class --- python/ray/tune/suggest/bohb.py | 376 +------------------------------- 1 file changed, 1 insertion(+), 375 deletions(-) diff --git a/python/ray/tune/suggest/bohb.py b/python/ray/tune/suggest/bohb.py index fa3b2093b707..51b413302d1a 100644 --- a/python/ray/tune/suggest/bohb.py +++ b/python/ray/tune/suggest/bohb.py @@ -4,384 +4,10 @@ from __future__ import division from __future__ import print_function -import ConfigSpace -import ConfigSpace.hyperparameters as CSH -import ConfigSpace.util -import numpy as np -import scipy.stats as sps -import statsmodels.api as sm - -from hpbandster.core.base_config_generator import base_config_generator +from hpbandster import BOHB from ray.tune.suggest import SuggestionAlgorithm -class BOHB(base_config_generator): - def __init__(self, - configspace, - min_points_in_model=None, - top_n_percent=15, - num_samples=64, - random_fraction=1 / 3, - bandwidth_factor=3, - min_bandwidth=1e-3, - **kwargs): - """ - Fits for each given budget a kernel density estimator on the best N percent of the - evaluated configurations on this budget. - - - Parameters: - ----------- - configspace: ConfigSpace - Configuration space object - top_n_percent: int - Determines the percentile of configurations that will be used as training data - for the kernel density estimator, e.g if set to 10 the 10% best configurations will be considered - for training. - min_points_in_model: int - minimum number of datapoints needed to fit a model - num_samples: int - number of samples drawn to optimize EI via sampling - random_fraction: float - fraction of random configurations returned - bandwidth_factor: float - widens the bandwidth for contiuous parameters for proposed points to optimize EI - min_bandwidth: float - to keep diversity, even when all (good) samples have the same value for one of the parameters, - a minimum bandwidth (Default: 1e-3) is used instead of zero. - - """ - super().__init__(**kwargs) - self.top_n_percent = top_n_percent - self.configspace = configspace - self.bw_factor = bandwidth_factor - self.min_bandwidth = min_bandwidth - - self.min_points_in_model = min_points_in_model - if min_points_in_model is None: - self.min_points_in_model = len( - self.configspace.get_hyperparameters()) + 1 - - if self.min_points_in_model < len( - self.configspace.get_hyperparameters()) + 1: - self.logger.warning( - 'Invalid min_points_in_model value. Setting it to %i' % - (len(self.configspace.get_hyperparameters()) + 1)) - self.min_points_in_model = len( - self.configspace.get_hyperparameters()) + 1 - - self.num_samples = num_samples - self.random_fraction = random_fraction - - hps = self.configspace.get_hyperparameters() - - self.kde_vartypes = "" - self.vartypes = [] - - for h in hps: - if hasattr(h, 'sequence'): - raise RuntimeError( - 'This version on BOHB does not support ordinal hyperparameters. Please encode %s as an integer parameter!' - % (h.name)) - - if hasattr(h, 'choices'): - self.kde_vartypes += 'u' - self.vartypes += [len(h.choices)] - else: - self.kde_vartypes += 'c' - self.vartypes += [0] - - self.vartypes = np.array(self.vartypes, dtype=int) - - # store precomputed probs for the categorical parameters - self.cat_probs = [] - - self.configs = dict() - self.losses = dict() - self.good_config_rankings = dict() - self.kde_models = dict() - - def largest_budget_with_model(self): - if len(self.kde_models) == 0: - return (-float('inf')) - return (max(self.kde_models.keys())) - - def get_config(self, *args): - """ - Function to sample a new configuration - - This function is called inside Hyperband to query a new configuration - - - returns: config - should return a valid configuration - - """ - - self.logger.debug('start sampling a new configuration.') - - sample = None - info_dict = {} - - # If no model is available, sample from prior - # also mix in a fraction of random configs - if len(self.kde_models.keys() - ) == 0 or np.random.rand() < self.random_fraction: - sample = self.configspace.sample_configuration() - info_dict['model_based_pick'] = False - - best = np.inf - best_vector = None - - if sample is None: - try: - - #sample from largest budget - budget = max(self.kde_models.keys()) - - l = self.kde_models[budget]['good'].pdf - g = self.kde_models[budget]['bad'].pdf - - minimize_me = lambda x: max(1e-32, g(x)) / max(l(x), 1e-32) - - kde_good = self.kde_models[budget]['good'] - kde_bad = self.kde_models[budget]['bad'] - - for i in range(self.num_samples): - idx = np.random.randint(0, len(kde_good.data)) - datum = kde_good.data[idx] - vector = [] - - for m, bw, t in zip(datum, kde_good.bw, self.vartypes): - - bw = max(bw, self.min_bandwidth) - if t == 0: - bw = self.bw_factor * bw - try: - vector.append( - sps.truncnorm.rvs( - -m / bw, (1 - m) / bw, loc=m, - scale=bw)) - except: - self.logger.warning( - "Truncated Normal failed for:\ndatum=%s\nbandwidth=%s\nfor entry with value %s" - % (datum, kde_good.bw, m)) - self.logger.warning( - "data in the KDE:\n%s" % kde_good.data) - else: - - if np.random.rand() < (1 - bw): - vector.append(int(m)) - else: - vector.append(np.random.randint(t)) - val = minimize_me(vector) - - if not np.isfinite(val): - self.logger.warning( - 'sampled vector: %s has EI value %s' % (vector, - val)) - self.logger.warning("data in the KDEs:\n%s\n%s" % - (kde_good.data, kde_bad.data)) - self.logger.warning("bandwidth of the KDEs:\n%s\n%s" % - (kde_good.bw, kde_bad.bw)) - self.logger.warning("l(x) = %s" % (l(vector))) - self.logger.warning("g(x) = %s" % (g(vector))) - - # right now, this happens because a KDE does not contain all values for a categorical parameter - # this cannot be fixed with the statsmodels KDE, so for now, we are just going to evaluate this one - # if the good_kde has a finite value, i.e. there is no config with that value in the bad kde, so it shouldn't be terrible. - if np.isfinite(l(vector)): - best_vector = vector - break - - if val < best: - best = val - best_vector = vector - - if best_vector is None: - self.logger.debug( - "Sampling based optimization with %i samples failed -> using random configuration" - % self.num_samples) - sample = self.configspace.sample_configuration( - ).get_dictionary() - info_dict['model_based_pick'] = False - else: - self.logger.debug('best_vector: {}, {}, {}, {}'.format( - best_vector, best, l(best_vector), g(best_vector))) - for i, hp_value in enumerate(best_vector): - if isinstance( - self.configspace.get_hyperparameter( - self.configspace.get_hyperparameter_by_idx( - i)), CSH.CategoricalHyperparameter): - best_vector[i] = int(np.rint(best_vector[i])) - sample = ConfigSpace.Configuration( - self.configspace, vector=best_vector).get_dictionary() - - try: - sample = ConfigSpace.util.deactivate_inactive_hyperparameters( - configuration_space=self.configspace, - configuration=sample) - info_dict['model_based_pick'] = True - - except Exception as e: - self.logger.warning(("="*50 + "\n")*3 +\ - "Error converting configuration:\n%s"%sample+\ - "\n here is a traceback:" +\ - traceback.format_exc()) - raise (e) - - except: - self.logger.warning( - "Sampling based optimization with %i samples failed\n %s \nUsing random configuration" - % (self.num_samples, traceback.format_exc())) - sample = self.configspace.sample_configuration() - info_dict['model_based_pick'] = False - - try: - sample = ConfigSpace.util.deactivate_inactive_hyperparameters( - configuration_space=self.configspace, - configuration=sample.get_dictionary()).get_dictionary() - except Exception as e: - self.logger.warning( - "Error (%s) converting configuration: %s -> " - "using random configuration!", e, sample) - sample = self.configspace.sample_configuration().get_dictionary() - self.logger.debug('done sampling a new configuration.') - return sample, info_dict - - def impute_conditional_data(self, array): - - return_array = np.empty_like(array) - - for i in range(array.shape[0]): - datum = np.copy(array[i]) - nan_indices = np.argwhere(np.isnan(datum)).flatten() - - while (np.any(nan_indices)): - nan_idx = nan_indices[0] - valid_indices = np.argwhere(np.isfinite( - array[:, nan_idx])).flatten() - - if len(valid_indices) > 0: - # pick one of them at random and overwrite all NaN values - row_idx = np.random.choice(valid_indices) - datum[nan_indices] = array[row_idx, nan_indices] - - else: - # no good point in the data has this value activated, so fill it with a valid but random value - t = self.vartypes[nan_idx] - if t == 0: - datum[nan_idx] = np.random.rand() - else: - datum[nan_idx] = np.random.randint(t) - - nan_indices = np.argwhere(np.isnan(datum)).flatten() - return_array[i, :] = datum - return (return_array) - - def new_result(self, job, update_model=True): - """ - function to register finished runs - - Every time a run has finished, this function should be called - to register it with the result logger. If overwritten, make - sure to call this method from the base class to ensure proper - logging. - - - Parameters: - ----------- - job: hpbandster.distributed.dispatcher.Job object - contains all the info about the run - """ - - super().new_result(job) - - if job.result is None: - # One could skip crashed results, but we decided to - # assign a +inf loss and count them as bad configurations - loss = np.inf - else: - # same for non numeric losses. - # Note that this means losses of minus infinity will count as bad! - loss = job.result["loss"] if np.isfinite( - job.result["loss"]) else np.inf - - budget = job.kwargs["budget"] - - if budget not in self.configs.keys(): - self.configs[budget] = [] - self.losses[budget] = [] - - # skip model building if we already have a bigger model - if max(list(self.kde_models.keys()) + [-np.inf]) > budget: - return - - # We want to get a numerical representation of the configuration in the original space - - conf = ConfigSpace.Configuration(self.configspace, - job.kwargs["config"]) - self.configs[budget].append(conf.get_array()) - self.losses[budget].append(loss) - - # skip model building: - # a) if not enough points are available - if len(self.configs[budget]) <= self.min_points_in_model - 1: - self.logger.debug( - "Only %i run(s) for budget %f available, need more than %s -> can't build model!" - % (len(self.configs[budget]), budget, - self.min_points_in_model + 1)) - return - - # b) during warnm starting when we feed previous results in and only update once - if not update_model: - return - - train_configs = np.array(self.configs[budget]) - train_losses = np.array(self.losses[budget]) - - n_good = max(self.min_points_in_model, - (self.top_n_percent * train_configs.shape[0]) // 100) - #n_bad = min(max(self.min_points_in_model, ((100-self.top_n_percent)*train_configs.shape[0])//100), 10) - n_bad = max( - self.min_points_in_model, - ((100 - self.top_n_percent) * train_configs.shape[0]) // 100) - - # Refit KDE for the current budget - idx = np.argsort(train_losses) - - train_data_good = self.impute_conditional_data( - train_configs[idx[:n_good]]) - train_data_bad = self.impute_conditional_data( - train_configs[idx[n_good:n_good + n_bad]]) - - if train_data_good.shape[0] <= train_data_good.shape[1]: - return - if train_data_bad.shape[0] <= train_data_bad.shape[1]: - return - - #more expensive crossvalidation method - #bw_estimation = 'cv_ls' - - # quick rule of thumb - bw_estimation = 'normal_reference' - - bad_kde = sm.nonparametric.KDEMultivariate( - data=train_data_bad, var_type=self.kde_vartypes, bw=bw_estimation) - good_kde = sm.nonparametric.KDEMultivariate( - data=train_data_good, var_type=self.kde_vartypes, bw=bw_estimation) - - bad_kde.bw = np.clip(bad_kde.bw, self.min_bandwidth, None) - good_kde.bw = np.clip(good_kde.bw, self.min_bandwidth, None) - - self.kde_models[budget] = {'good': good_kde, 'bad': bad_kde} - - # update probs for the categorical parameters for later sampling - self.logger.debug( - 'done building a new model for budget %f based on %i/%i split\nBest loss for this budget:%f\n\n\n\n\n' - % (budget, n_good, n_bad, np.min(train_losses))) - - class JobWrapper(): """Dummy class""" From 1eb96747769ce4a084b752dd52d9c6d6df3eba4b Mon Sep 17 00:00:00 2001 From: Lisa Dunlap Date: Fri, 9 Aug 2019 01:11:32 -0700 Subject: [PATCH 08/31] fixed docs --- doc/source/tune-searchalg.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/tune-searchalg.rst b/doc/source/tune-searchalg.rst index d249d27b415a..4a52a0667881 100644 --- a/doc/source/tune-searchalg.rst +++ b/doc/source/tune-searchalg.rst @@ -183,7 +183,7 @@ An example of this can be found in `ax_example.py `__ to perform sequential model-based hyperparameter optimization in conjunction with Hyperband. Note that this class does not extend ``ray.tune.suggest.BasicVariantGenerator``, so you will not be able to use Tune's default variant generation/search space declaration when using BOHB. Also note that BOHB is intended to be paried with the Hyperband scheduler `hp_bohb.py `__. From fe2e96248d633b06bfd18cd361def1e795df2059 Mon Sep 17 00:00:00 2001 From: Lisa Dunlap Date: Fri, 9 Aug 2019 02:15:27 -0700 Subject: [PATCH 09/31] fixed PR comments and added class descriptions --- ci/jenkins_tests/run_tune_tests.sh | 4 ++ python/ray/tune/examples/bohb_example.py | 12 ++--- python/ray/tune/schedulers/hb_bohb.py | 66 ++++++++++++------------ python/ray/tune/suggest/bohb.py | 65 ++++++++++++++++++----- 4 files changed, 95 insertions(+), 52 deletions(-) diff --git a/ci/jenkins_tests/run_tune_tests.sh b/ci/jenkins_tests/run_tune_tests.sh index 9368930b6043..eb57fcaf07b7 100755 --- a/ci/jenkins_tests/run_tune_tests.sh +++ b/ci/jenkins_tests/run_tune_tests.sh @@ -103,3 +103,7 @@ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ python /ray/python/ray/tune/examples/skopt_example.py \ --smoke-test + +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ + python /ray/python/ray/tune/examples/bohb_example.py \ + --smoke-test diff --git a/python/ray/tune/examples/bohb_example.py b/python/ray/tune/examples/bohb_example.py index a37dc0ed1320..cf7df316b901 100644 --- a/python/ray/tune/examples/bohb_example.py +++ b/python/ray/tune/examples/bohb_example.py @@ -7,15 +7,12 @@ import argparse import json import os -import random import numpy as np -import ConfigSpace -import ConfigSpace.hyperparameters as CSH import ConfigSpace.util import ray -from ray.tune import Trainable, run, sample_from +from ray.tune import Trainable, run from ray.tune.schedulers import HyperBandForBOHB from ray.tune.suggest import TuneBOHB @@ -69,9 +66,9 @@ def _restore(self, checkpoint_path): CS = ConfigSpace config_space = CS.ConfigurationSpace() config_space.add_hyperparameter( - CS.UniformFloatHyperparameter('height', lower=10, upper=100)) + CS.UniformFloatHyperparameter("height", lower=10, upper=100)) config_space.add_hyperparameter( - CS.UniformFloatHyperparameter('width', lower=0, upper=100)) + CS.UniformFloatHyperparameter("width", lower=0, upper=100)) bohb = HyperBandForBOHB( time_attr="training_iteration", @@ -79,7 +76,8 @@ def _restore(self, checkpoint_path): grace_period=5, max_t=100) - run(MyTrainableClass, + run( + MyTrainableClass, name="bohb_test", scheduler=bohb, search_alg=TuneBOHB( diff --git a/python/ray/tune/schedulers/hb_bohb.py b/python/ray/tune/schedulers/hb_bohb.py index c4052d25137a..10adb9e50e88 100644 --- a/python/ray/tune/schedulers/hb_bohb.py +++ b/python/ray/tune/schedulers/hb_bohb.py @@ -2,11 +2,10 @@ from __future__ import division from __future__ import print_function -import collections import numpy as np import logging -from ray.tune.schedulers.trial_scheduler import FIFOScheduler, TrialScheduler +from ray.tune.schedulers.trial_scheduler import TrialScheduler from ray.tune.schedulers.hyperband import HyperBandScheduler, Bracket from ray.tune.trial import Trial @@ -14,9 +13,9 @@ class HyperBandForBOHB(HyperBandScheduler): - """Implements the HyperBand early stopping algorithm. + """Extends HyperBand early stopping algorithm for BOHB. - @!!!!!!!!!!!!!! Key changes: + Key changes: Bracket filling is reversed Trial results @@ -26,7 +25,7 @@ class HyperBandForBOHB(HyperBandScheduler): Note that you can pass in something non-temporal such as `training_iteration` as a measure of progress, the only requirement is that the attribute should increase monotonically. - reward_attr (str): The training result objective value attribute. As + metric (str): The training result objective value attribute. As with `time_attr`, this may refer to any objective value. Stopping procedures will use this attribute. max_t (int): max time units per trial. Trials will be stopped after @@ -34,6 +33,8 @@ class HyperBandForBOHB(HyperBandScheduler): The scheduler will terminate trials after this time has passed. Note that this is different from the semantics of `max_t` as mentioned in the original HyperBand paper. + mode (str): One of {min, max}. Determines whether objective is + minimizing or maximizing the metric attribute. """ def on_trial_add(self, trial_runner, trial): @@ -48,7 +49,6 @@ def on_trial_add(self, trial_runner, trial): cur_band = self._hyperbands[self._state["band_idx"]] if cur_bracket is None or cur_bracket.filled(): print("Adding a new bracket.") - # import ipdb; ipdb.set_trace() retry = True while retry: # if current iteration is filled, create new iteration @@ -57,19 +57,21 @@ def on_trial_add(self, trial_runner, trial): self._hyperbands.append(cur_band) self._state["band_idx"] += 1 - #### MAIN CHANGE HERE - largest bracket first! + # MAIN CHANGE HERE - largest bracket first! # cur_band will always be less than s_max_1 or else filled s = self._s_max_1 - len(cur_band) - 1 assert s >= 0, "Current band is filled!" - #### MAIN CHANGE HERE! + # MAIN CHANGE HERE! if self._get_r0(s) == 0: logger.info("Bracket too small - Retrying...") cur_bracket = None else: retry = False - cur_bracket = ContinuationBracket(self._time_attr, self._get_n0(s), - self._get_r0(s), self._max_t_attr, - self._eta, s) + cur_bracket = ContinuationBracket(self._time_attr, + self._get_n0(s), + self._get_r0(s), + self._max_t_attr, + self._eta, s) cur_band.append(cur_bracket) self._state["bracket"] = cur_bracket @@ -95,19 +97,17 @@ def on_trial_result(self, trial_runner, trial, result): result["hyperband_info"]["budget"] = bracket._cumul_r - #### MAIN CHANGE HERE! + # MAIN CHANGE HERE! statuses = [(t, t.status) for t in bracket._live_trials] - if not bracket.filled() or any( - status != Trial.PAUSED for t, status in statuses - if t is not trial): - print(Counter([status for _, status in statuses])) + if not bracket.filled() or any(status != Trial.PAUSED + for t, status in statuses + if t is not trial): trial_runner._search_alg.on_pause(trial.trial_id) return TrialScheduler.PAUSE - # import ipdb; ipdb.set_trace() action = self._process_bracket(trial_runner, bracket, trial) return action - def _process_bracket(self, trial_runner, bracket, trial): + def _process_bracket(self, trial_runner, bracket): """This is called whenever a trial makes progress. When all live trials in the bracket have no more iterations left, @@ -122,7 +122,8 @@ def _process_bracket(self, trial_runner, bracket, trial): bracket.cleanup_full(trial_runner) return TrialScheduler.STOP - good, bad = bracket.successive_halving(self._reward_attr) + good, bad = bracket.successive_halving( + self._metric_op * self._metric) # kill bad trials self._num_stopped += len(bad) for t in bad: @@ -141,9 +142,9 @@ def _process_bracket(self, trial_runner, bracket, trial): if bracket.continue_trial(t): if t.status == Trial.PAUSED: trial_runner.trial_executor.unpause_trial(t) - #### MAIN CHANGE HERE! + # MAIN CHANGE HERE! trial_runner._search_alg.on_unpause(t.trial_id) - #### MAIN CHANGE HERE! + # MAIN CHANGE HERE! elif t.status == Trial.RUNNING: action = TrialScheduler.CONTINUE return action @@ -165,16 +166,18 @@ def choose_trial_to_run(self, trial_runner): if (trial.status == Trial.PENDING and trial_runner.has_resources(trial.resources)): return trial - #### MAIN CHANGE HERE! - if not any(t.status == Trial.RUNNING for t in trial_runner.get_trials()): + # MAIN CHANGE HERE! + if not any(t.status == Trial.RUNNING + for t in trial_runner.get_trials()): for hyperband in self._hyperbands: for bracket in hyperband: - if bracket and any( - trial.status == Trial.PAUSED for trial in bracket.current_trials()): + if bracket and any(trial.status == Trial.PAUSED + for trial in bracket.current_trials()): self._process_bracket(trial_runner, bracket, None) - #### MAIN CHANGE HERE! + # MAIN CHANGE HERE! return None + class ContinuationBracket(Bracket): """Logical object for tracking Hyperband bracket progress. Keeps track of proper parameters as designated by HyperBand. @@ -182,7 +185,7 @@ class ContinuationBracket(Bracket): Also keeps track of progress to ensure good scheduling. """ - def successive_halving(self, reward_attr): + def successive_halving(self, metric): assert self._halves > 0 self._halves -= 1 self._n /= self._eta @@ -193,19 +196,16 @@ def successive_halving(self, reward_attr): # MAIN CHANGE HERE - don't accumulate R. self._cumul_r = self._r - ################### + sorted_trials = sorted( - self._live_trials, key=lambda t: self._live_trials[t][reward_attr]) + self._live_trials, key=lambda t: self._live_trials[t][metric]) good, bad = sorted_trials[-self._n:], sorted_trials[:-self._n] return good, bad def update_trial_stats(self, trial, result): """Update result for trial. Called after trial has finished - an iteration - will decrement iteration count. - - TODO(rliaw): The other alternative is to keep the trials - in and make sure they're not set as pending later.""" + an iteration - will decrement iteration count.""" assert trial in self._live_trials assert self._get_result_time(result) >= 0 diff --git a/python/ray/tune/suggest/bohb.py b/python/ray/tune/suggest/bohb.py index 51b413302d1a..91e3b9c4b84c 100644 --- a/python/ray/tune/suggest/bohb.py +++ b/python/ray/tune/suggest/bohb.py @@ -4,7 +4,11 @@ from __future__ import division from __future__ import print_function -from hpbandster import BOHB +try: + from hpbandster import BOHB +except ImportError: + hpbandster = None + from ray.tune.suggest import SuggestionAlgorithm @@ -16,19 +20,58 @@ def __init__(self, loss, budget, config): self.kwargs = {"budget": budget, "config": config.copy()} self.exception = None + class TuneBOHB(SuggestionAlgorithm): - """Tune Suggestion Algorithm for BOHB code""" + """Tune Suggestion Algorithm for BOHB code + + Requires HpBandSter and ConfigSpace to be installed. You can install + HpBandSter with the command: `pip install hpbandster` and + ConfigSpace with the command: `pip install ConfigSpace`. + + Parameters: + space (dict): Continuous ConfigSpace search space. Parameters will + be sampled from this space which will be used to run trials. + max_concurrent (int): Number of maximum concurrent trials. Defaults + to 10. + metric (str): The training result objective value attribute. + mode (str): One of {min, max}. Determines whether objective is + minimizing or maximizing the metric attribute. + + Example: + >>> CS = ConfigSpace + >>> config_space = CS.ConfigurationSpace() + >>> config_space.add_hyperparameter( + >>> CS.UniformFloatHyperparameter('width', lower=0, upper=20)) + >>> config_space.add_hyperparameter( + >>> CS.UniformFloatHyperparameter('height', lower=-100, upper=100)) + >>> config_space.add_hyperparameter( + >>> CS.CategoricalHyperparameter(name='activation', choices=['relu', 'tanh'])) + >>> current_best_params = [{ + >>> 'width': 10, + >>> 'height': 0, + >>> 'activation': 0, # The index of "relu" + >>> }] + >>> algo = TuneBOHB( + >>> config_space, max_concurrent=4, metric="mean_loss", mode="min") + """ def __init__(self, space, - max_concurrent=8, + max_concurrent=10, metric="neg_mean_loss", - bohb_config=None): + bohb_config=None, + mode="max"): + assert hpbandster is not None, "HpBandSter must be installed!" + assert mode in ["min", "max"], "`mode` must be 'min' or 'max'!" self._max_concurrent = max_concurrent self.trial_to_params = {} self.running = set() self.paused = set() self.metric = metric + if mode == "max": + self._metric_op = -1. + elif mode == "min": + self._metric_op = 1. bohb_config = bohb_config or {} self.bohber = BOHB(space, **bohb_config) super(TuneBOHB, self).__init__() @@ -46,7 +89,6 @@ def on_trial_result(self, trial_id, result): self.running.add(trial_id) if "budget" in result.get("hyperband_info", {}): hbs_wrapper = self.to_wrapper(trial_id, result) - print("adding new result", vars(hbs_wrapper)) self.bohber.new_result(hbs_wrapper) def on_trial_complete(self, @@ -57,16 +99,15 @@ def on_trial_complete(self, del self.trial_to_params[trial_id] if trial_id in self.paused: self.paused.remove(trial_id) - elif trial_id in self.running: + if trial_id in self.running: self.running.remove(trial_id) - else: - import ipdb; ipdb.set_trace() - def to_wrapper(self, trial_id, result): return JobWrapper( - -result[self.metric], result["hyperband_info"]["budget"], - {k: result["config"][k] for k in self.trial_to_params[trial_id]}) + self._metric_op * result[self.metric], + result["hyperband_info"]["budget"], + {k: result["config"][k] + for k in self.trial_to_params[trial_id]}) def on_pause(self, trial_id): self.paused.add(trial_id) @@ -74,4 +115,4 @@ def on_pause(self, trial_id): def on_unpause(self, trial_id): self.paused.remove(trial_id) - self.running.add(trial_id) \ No newline at end of file + self.running.add(trial_id) From c109e1319a3273fd19a392edba3e3bbdb08ef493 Mon Sep 17 00:00:00 2001 From: Lisa Dunlap Date: Fri, 9 Aug 2019 10:03:38 -0700 Subject: [PATCH 10/31] fixed import error --- python/ray/tune/examples/bohb_example.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/ray/tune/examples/bohb_example.py b/python/ray/tune/examples/bohb_example.py index cf7df316b901..92e177729da9 100644 --- a/python/ray/tune/examples/bohb_example.py +++ b/python/ray/tune/examples/bohb_example.py @@ -9,7 +9,6 @@ import os import numpy as np -import ConfigSpace.util import ray from ray.tune import Trainable, run @@ -48,6 +47,9 @@ def _restore(self, checkpoint_path): if __name__ == "__main__": + import argparse + import ConfigSpace + parser = argparse.ArgumentParser() parser.add_argument( "--smoke-test", action="store_true", help="Finish quickly for testing") From bc3a95151b30a4892055eb7c70cce1ccd1f25d6d Mon Sep 17 00:00:00 2001 From: Lisa Dunlap Date: Fri, 9 Aug 2019 10:04:34 -0700 Subject: [PATCH 11/31] formatting --- python/ray/tune/examples/bohb_example.py | 1 - python/ray/tune/schedulers/hb_bohb.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/python/ray/tune/examples/bohb_example.py b/python/ray/tune/examples/bohb_example.py index 92e177729da9..b097fc7d940e 100644 --- a/python/ray/tune/examples/bohb_example.py +++ b/python/ray/tune/examples/bohb_example.py @@ -4,7 +4,6 @@ from __future__ import division from __future__ import print_function -import argparse import json import os diff --git a/python/ray/tune/schedulers/hb_bohb.py b/python/ray/tune/schedulers/hb_bohb.py index 10adb9e50e88..1769883b58d1 100644 --- a/python/ray/tune/schedulers/hb_bohb.py +++ b/python/ray/tune/schedulers/hb_bohb.py @@ -87,7 +87,7 @@ def on_trial_result(self, trial_runner, trial, result): This scheduler will not start trials but will stop trials. The current running trial will not be handled, as the trialrunner will be given control to handle it.""" - from collections import Counter + result["hyperband_info"] = {} bracket, _ = self._trial_info[trial] bracket.update_trial_stats(trial, result) From f405d936cc2c5f6394acc8a5aa0488443b62a27e Mon Sep 17 00:00:00 2001 From: Lisa Dunlap Date: Fri, 9 Aug 2019 10:40:01 -0700 Subject: [PATCH 12/31] reformatting --- python/ray/tune/examples/bohb_example.py | 29 ++++++++++++++---------- python/ray/tune/schedulers/hb_bohb.py | 2 +- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/python/ray/tune/examples/bohb_example.py b/python/ray/tune/examples/bohb_example.py index b097fc7d940e..b307c34d57ce 100644 --- a/python/ray/tune/examples/bohb_example.py +++ b/python/ray/tune/examples/bohb_example.py @@ -77,17 +77,7 @@ def _restore(self, checkpoint_path): grace_period=5, max_t=100) - run( - MyTrainableClass, - name="bohb_test", - scheduler=bohb, - search_alg=TuneBOHB( - config_space, - max_concurrent=4, - # num_concurrent=100, - metric="episode_reward_mean", - ), - **{ + config = { "stop": { "training_iteration": 1 if args.smoke_test else 99999 }, @@ -96,4 +86,19 @@ def _restore(self, checkpoint_path): "cpu": 1, "gpu": 0 }, - }) + } + + algo = TuneBOHB( + config_space, + max_concurrent=4, + # num_concurrent=100, + metric="mean_loss", + mode="min" + ) + + run( + MyTrainableClass, + name="bohb_test", + scheduler=bohb, + search_alg=algo, + **config) diff --git a/python/ray/tune/schedulers/hb_bohb.py b/python/ray/tune/schedulers/hb_bohb.py index 1769883b58d1..f1767a47dc6e 100644 --- a/python/ray/tune/schedulers/hb_bohb.py +++ b/python/ray/tune/schedulers/hb_bohb.py @@ -87,7 +87,7 @@ def on_trial_result(self, trial_runner, trial, result): This scheduler will not start trials but will stop trials. The current running trial will not be handled, as the trialrunner will be given control to handle it.""" - + result["hyperband_info"] = {} bracket, _ = self._trial_info[trial] bracket.update_trial_stats(trial, result) From 7e5cabfec995a4826df715d626e06b367d92a971 Mon Sep 17 00:00:00 2001 From: Lisa Dunlap Date: Fri, 9 Aug 2019 14:26:53 -0500 Subject: [PATCH 13/31] Update doc/source/tune-searchalg.rst Co-Authored-By: Richard Liaw --- doc/source/tune-searchalg.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/tune-searchalg.rst b/doc/source/tune-searchalg.rst index 4a52a0667881..7f017ce2bde2 100644 --- a/doc/source/tune-searchalg.rst +++ b/doc/source/tune-searchalg.rst @@ -185,7 +185,7 @@ An example of this can be found in `ax_example.py `__ to perform sequential model-based hyperparameter optimization in conjunction with Hyperband. Note that this class does not extend ``ray.tune.suggest.BasicVariantGenerator``, so you will not be able to use Tune's default variant generation/search space declaration when using BOHB. Also note that BOHB is intended to be paried with the Hyperband scheduler `hp_bohb.py `__. +``BOHB`` (Bayesian Optimization HyperBand) is a SearchAlgorithm that is backed by `HpBandSter `__ to perform sequential model-based hyperparameter optimization in conjunction with HyperBand. Note that this class does not extend ``ray.tune.suggest.BasicVariantGenerator``, so you will not be able to use Tune's default variant generation/search space declaration when using BOHB. Also note that BOHB is intended to be paired with a specific Hyperband scheduler `ray.tune.schedulers.HyperBandForBOHB` `__. In order to use this search algorithm, you will need to install HpBandSter and ConfigSpace via the following command: From c19f5ce7b6d27f65e2713ec572b040a7e32a096b Mon Sep 17 00:00:00 2001 From: Lisa Dunlap Date: Fri, 9 Aug 2019 12:31:33 -0700 Subject: [PATCH 14/31] added imports to dockerfile --- doc/source/tune-searchalg.rst | 4 ++-- docker/examples/Dockerfile | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/source/tune-searchalg.rst b/doc/source/tune-searchalg.rst index 7f017ce2bde2..8fca40cc4650 100644 --- a/doc/source/tune-searchalg.rst +++ b/doc/source/tune-searchalg.rst @@ -182,8 +182,8 @@ An example of this can be found in `ax_example.py `__ to perform sequential model-based hyperparameter optimization in conjunction with HyperBand. Note that this class does not extend ``ray.tune.suggest.BasicVariantGenerator``, so you will not be able to use Tune's default variant generation/search space declaration when using BOHB. Also note that BOHB is intended to be paired with a specific Hyperband scheduler `ray.tune.schedulers.HyperBandForBOHB` `__. diff --git a/docker/examples/Dockerfile b/docker/examples/Dockerfile index bafcdf35e628..4b488927af04 100644 --- a/docker/examples/Dockerfile +++ b/docker/examples/Dockerfile @@ -14,5 +14,7 @@ RUN pip install --upgrade git+git://github.com/hyperopt/hyperopt.git RUN pip install --upgrade sigopt RUN pip install --upgrade nevergrad RUN pip install --upgrade scikit-optimize +RUN pip install --upgrade hpbandster +RUN pip install --upgrade ConfigSpace RUN pip install -U pytest-remotedata>=0.3.1 RUN conda install pytorch-cpu torchvision-cpu -c pytorch From fa2159d28bf588f1f07ffd7c8b912cff99c40640 Mon Sep 17 00:00:00 2001 From: Lisa Dunlap Date: Sat, 10 Aug 2019 15:52:14 -0700 Subject: [PATCH 15/31] yet another import error --- python/ray/tune/examples/bohb_example.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/ray/tune/examples/bohb_example.py b/python/ray/tune/examples/bohb_example.py index b307c34d57ce..e64e51206b19 100644 --- a/python/ray/tune/examples/bohb_example.py +++ b/python/ray/tune/examples/bohb_example.py @@ -11,8 +11,8 @@ import ray from ray.tune import Trainable, run -from ray.tune.schedulers import HyperBandForBOHB -from ray.tune.suggest import TuneBOHB +from ray.tune.schedulers.hb_bohb import HyperBandForBOHB +from ray.tune.suggest.bohb import TuneBOHB class MyTrainableClass(Trainable): From 06c6ddae375b43364e469035cd73b1d38b3b23c9 Mon Sep 17 00:00:00 2001 From: Lisa Dunlap Date: Sun, 11 Aug 2019 15:20:29 -0700 Subject: [PATCH 16/31] fix --- python/ray/tune/examples/bohb_example.py | 2 -- python/ray/tune/suggest/bohb.py | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/python/ray/tune/examples/bohb_example.py b/python/ray/tune/examples/bohb_example.py index e64e51206b19..5df79c029a9e 100644 --- a/python/ray/tune/examples/bohb_example.py +++ b/python/ray/tune/examples/bohb_example.py @@ -74,7 +74,6 @@ def _restore(self, checkpoint_path): bohb = HyperBandForBOHB( time_attr="training_iteration", metric="episode_reward_mean", - grace_period=5, max_t=100) config = { @@ -91,7 +90,6 @@ def _restore(self, checkpoint_path): algo = TuneBOHB( config_space, max_concurrent=4, - # num_concurrent=100, metric="mean_loss", mode="min" ) diff --git a/python/ray/tune/suggest/bohb.py b/python/ray/tune/suggest/bohb.py index 91e3b9c4b84c..ece0c27ef9a3 100644 --- a/python/ray/tune/suggest/bohb.py +++ b/python/ray/tune/suggest/bohb.py @@ -34,6 +34,7 @@ class TuneBOHB(SuggestionAlgorithm): max_concurrent (int): Number of maximum concurrent trials. Defaults to 10. metric (str): The training result objective value attribute. + bohb_config (dict): configuration for HpBandSter BOHB algorithm mode (str): One of {min, max}. Determines whether objective is minimizing or maximizing the metric attribute. From 922af5f573f3b87a4cb725a694a63192fe1d6223 Mon Sep 17 00:00:00 2001 From: Lisa Dunlap Date: Sun, 11 Aug 2019 15:48:43 -0700 Subject: [PATCH 17/31] format --- python/ray/tune/examples/bohb_example.py | 29 ++++++++++-------------- python/ray/tune/suggest/bohb.py | 3 ++- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/python/ray/tune/examples/bohb_example.py b/python/ray/tune/examples/bohb_example.py index 5df79c029a9e..9390a73c442e 100644 --- a/python/ray/tune/examples/bohb_example.py +++ b/python/ray/tune/examples/bohb_example.py @@ -77,25 +77,20 @@ def _restore(self, checkpoint_path): max_t=100) config = { - "stop": { - "training_iteration": 1 if args.smoke_test else 99999 - }, - "num_samples": 20, - "resources_per_trial": { - "cpu": 1, - "gpu": 0 - }, - } + "stop": { + "training_iteration": 1 if args.smoke_test else 99999 + }, + "num_samples": 20, + "resources_per_trial": { + "cpu": 1, + "gpu": 0 + }, + } algo = TuneBOHB( - config_space, - max_concurrent=4, - metric="mean_loss", - mode="min" - ) - - run( - MyTrainableClass, + config_space, max_concurrent=4, metric="mean_loss", mode="min") + + run(MyTrainableClass, name="bohb_test", scheduler=bohb, search_alg=algo, diff --git a/python/ray/tune/suggest/bohb.py b/python/ray/tune/suggest/bohb.py index ece0c27ef9a3..88480878a61c 100644 --- a/python/ray/tune/suggest/bohb.py +++ b/python/ray/tune/suggest/bohb.py @@ -46,7 +46,8 @@ class TuneBOHB(SuggestionAlgorithm): >>> config_space.add_hyperparameter( >>> CS.UniformFloatHyperparameter('height', lower=-100, upper=100)) >>> config_space.add_hyperparameter( - >>> CS.CategoricalHyperparameter(name='activation', choices=['relu', 'tanh'])) + >>> CS.CategoricalHyperparameter(name='actuvation', + >>> choices=['relu', 'tanh'])) >>> current_best_params = [{ >>> 'width': 10, >>> 'height': 0, From be0b1105197a2d43db3aacd98a24da2aff2677dc Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Sun, 11 Aug 2019 16:17:25 -0700 Subject: [PATCH 18/31] increase redundancy between HB and BOHB --- python/ray/tune/examples/bohb_example.py | 4 +- python/ray/tune/schedulers/__init__.py | 3 +- python/ray/tune/schedulers/hb_bohb.py | 126 ++++------------------- python/ray/tune/schedulers/hyperband.py | 7 +- python/ray/tune/suggest/bohb.py | 79 ++++++++------ 5 files changed, 74 insertions(+), 145 deletions(-) diff --git a/python/ray/tune/examples/bohb_example.py b/python/ray/tune/examples/bohb_example.py index 5df79c029a9e..aada30d65952 100644 --- a/python/ray/tune/examples/bohb_example.py +++ b/python/ray/tune/examples/bohb_example.py @@ -71,7 +71,7 @@ def _restore(self, checkpoint_path): config_space.add_hyperparameter( CS.UniformFloatHyperparameter("width", lower=0, upper=100)) - bohb = HyperBandForBOHB( + bohb_hyperband = HyperBandForBOHB( time_attr="training_iteration", metric="episode_reward_mean", max_t=100) @@ -97,6 +97,6 @@ def _restore(self, checkpoint_path): run( MyTrainableClass, name="bohb_test", - scheduler=bohb, + scheduler=bohb_hyperband, search_alg=algo, **config) diff --git a/python/ray/tune/schedulers/__init__.py b/python/ray/tune/schedulers/__init__.py index 34655372f40a..3731724c92ea 100644 --- a/python/ray/tune/schedulers/__init__.py +++ b/python/ray/tune/schedulers/__init__.py @@ -4,6 +4,7 @@ from ray.tune.schedulers.trial_scheduler import TrialScheduler, FIFOScheduler from ray.tune.schedulers.hyperband import HyperBandScheduler +from ray.tune.schedulers.hb_bohb import HyperBandForBOHB from ray.tune.schedulers.async_hyperband import (AsyncHyperBandScheduler, ASHAScheduler) from ray.tune.schedulers.median_stopping_rule import MedianStoppingRule @@ -12,5 +13,5 @@ __all__ = [ "TrialScheduler", "HyperBandScheduler", "AsyncHyperBandScheduler", "ASHAScheduler", "MedianStoppingRule", "FIFOScheduler", - "PopulationBasedTraining" + "PopulationBasedTraining", "HyperBandForBOHB" ] diff --git a/python/ray/tune/schedulers/hb_bohb.py b/python/ray/tune/schedulers/hb_bohb.py index f1767a47dc6e..3871a91c93d3 100644 --- a/python/ray/tune/schedulers/hb_bohb.py +++ b/python/ray/tune/schedulers/hb_bohb.py @@ -8,6 +8,7 @@ from ray.tune.schedulers.trial_scheduler import TrialScheduler from ray.tune.schedulers.hyperband import HyperBandScheduler, Bracket from ray.tune.trial import Trial +from ray.tune.error import TuneError logger = logging.getLogger(__name__) @@ -15,26 +16,13 @@ class HyperBandForBOHB(HyperBandScheduler): """Extends HyperBand early stopping algorithm for BOHB. - Key changes: - Bracket filling is reversed - Trial results - - - Args: - time_attr (str): The training result attr to use for comparing time. - Note that you can pass in something non-temporal such as - `training_iteration` as a measure of progress, the only requirement - is that the attribute should increase monotonically. - metric (str): The training result objective value attribute. As - with `time_attr`, this may refer to any objective value. Stopping - procedures will use this attribute. - max_t (int): max time units per trial. Trials will be stopped after - max_t time units (determined by time_attr) have passed. - The scheduler will terminate trials after this time has passed. - Note that this is different from the semantics of `max_t` as - mentioned in the original HyperBand paper. - mode (str): One of {min, max}. Determines whether objective is - minimizing or maximizing the metric attribute. + The default HyperBandScheduler implements pipelining for efficiency. This + class introduces key changes: + - Trials are now placed so that the bracket with the largest size + is filled first. + - Trials will be paused even if the bracket is not filled. This allows + BOHB to insert new trials into the training. + """ def on_trial_add(self, trial_runner, trial): @@ -43,12 +31,12 @@ def on_trial_add(self, trial_runner, trial): On a new trial add, if current bracket is not filled, add to current bracket. Else, if current band is not filled, create new bracket, add to current bracket. - Else, create new iteration, create new bracket, add to bracket.""" + Else, create new iteration, create new bracket, add to bracket. + """ cur_bracket = self._state["bracket"] cur_band = self._hyperbands[self._state["band_idx"]] if cur_bracket is None or cur_bracket.filled(): - print("Adding a new bracket.") retry = True while retry: # if current iteration is filled, create new iteration @@ -67,11 +55,13 @@ def on_trial_add(self, trial_runner, trial): cur_bracket = None else: retry = False - cur_bracket = ContinuationBracket(self._time_attr, - self._get_n0(s), - self._get_r0(s), - self._max_t_attr, - self._eta, s) + cur_bracket = Bracket( + self._time_attr, + self._get_n0(s), + self._get_r0(s), + self._max_t_attr, + self._eta, + s) cur_band.append(cur_bracket) self._state["bracket"] = cur_bracket @@ -107,47 +97,9 @@ def on_trial_result(self, trial_runner, trial, result): action = self._process_bracket(trial_runner, bracket, trial) return action - def _process_bracket(self, trial_runner, bracket): - """This is called whenever a trial makes progress. - - When all live trials in the bracket have no more iterations left, - Trials will be successively halved. If bracket is done, all - non-running trials will be stopped and cleaned up, - and during each halving phase, bad trials will be stopped while good - trials will return to "PENDING".""" - - action = TrialScheduler.PAUSE - if bracket.cur_iter_done(): - if bracket.finished(): - bracket.cleanup_full(trial_runner) - return TrialScheduler.STOP - - good, bad = bracket.successive_halving( - self._metric_op * self._metric) - # kill bad trials - self._num_stopped += len(bad) - for t in bad: - if t.status == Trial.PAUSED: - trial_runner.stop_trial(t) - elif t.status == Trial.RUNNING: - bracket.cleanup_trial(t) - action = TrialScheduler.STOP - else: - raise Exception("Trial with unexpected status encountered") - - # ready the good trials - if trial is too far ahead, don't continue - for t in good: - if t.status not in [Trial.PAUSED, Trial.RUNNING]: - raise Exception("Trial with unexpected status encountered") - if bracket.continue_trial(t): - if t.status == Trial.PAUSED: - trial_runner.trial_executor.unpause_trial(t) - # MAIN CHANGE HERE! - trial_runner._search_alg.on_unpause(t.trial_id) - # MAIN CHANGE HERE! - elif t.status == Trial.RUNNING: - action = TrialScheduler.CONTINUE - return action + def _unpause_trial(self, trial_runner, trial): + trial_runner.trial_executor.unpause_trial(trial) + trial_runner._search_alg.on_unpause(trial.trial_id) def choose_trial_to_run(self, trial_runner): """Fair scheduling within iteration by completion percentage. @@ -176,41 +128,3 @@ def choose_trial_to_run(self, trial_runner): self._process_bracket(trial_runner, bracket, None) # MAIN CHANGE HERE! return None - - -class ContinuationBracket(Bracket): - """Logical object for tracking Hyperband bracket progress. Keeps track - of proper parameters as designated by HyperBand. - - Also keeps track of progress to ensure good scheduling. - """ - - def successive_halving(self, metric): - assert self._halves > 0 - self._halves -= 1 - self._n /= self._eta - self._n = int(np.ceil(self._n)) - - self._r *= self._eta - self._r = int(min(self._r, self._max_t_attr - self._cumul_r)) - - # MAIN CHANGE HERE - don't accumulate R. - self._cumul_r = self._r - - sorted_trials = sorted( - self._live_trials, key=lambda t: self._live_trials[t][metric]) - - good, bad = sorted_trials[-self._n:], sorted_trials[:-self._n] - return good, bad - - def update_trial_stats(self, trial, result): - """Update result for trial. Called after trial has finished - an iteration - will decrement iteration count.""" - - assert trial in self._live_trials - assert self._get_result_time(result) >= 0 - - delta = self._get_result_time(result) - \ - self._get_result_time(self._live_trials[trial]) - self._completed_progress += delta - self._live_trials[trial] = result diff --git a/python/ray/tune/schedulers/hyperband.py b/python/ray/tune/schedulers/hyperband.py index b1ca1deec11b..6bd682cedb6c 100644 --- a/python/ray/tune/schedulers/hyperband.py +++ b/python/ray/tune/schedulers/hyperband.py @@ -210,7 +210,7 @@ def _process_bracket(self, trial_runner, bracket, trial): raise Exception("Trial with unexpected status encountered") if bracket.continue_trial(t): if t.status == Trial.PAUSED: - trial_runner.trial_executor.unpause_trial(t) + self._unpause_trial(trial_runner, t) elif t.status == Trial.RUNNING: action = TrialScheduler.CONTINUE return action @@ -279,6 +279,9 @@ def debug_string(self): out += "\n {}".format(bracket) return out + def _unpause_trial(self, trial_runner, trial): + trial_runner.trial_executor.unpause_trial(trial) + class Bracket(): """Logical object for tracking Hyperband bracket progress. Keeps track @@ -349,7 +352,7 @@ def successive_halving(self, metric, metric_op): self._r *= self._eta self._r = int(min(self._r, self._max_t_attr - self._cumul_r)) - self._cumul_r += self._r + self._cumul_r = self._r sorted_trials = sorted( self._live_trials, key=lambda t: metric_op * self._live_trials[t][metric]) diff --git a/python/ray/tune/suggest/bohb.py b/python/ray/tune/suggest/bohb.py index ece0c27ef9a3..d2ca83a03769 100644 --- a/python/ray/tune/suggest/bohb.py +++ b/python/ray/tune/suggest/bohb.py @@ -22,45 +22,51 @@ def __init__(self, loss, budget, config): class TuneBOHB(SuggestionAlgorithm): - """Tune Suggestion Algorithm for BOHB code - - Requires HpBandSter and ConfigSpace to be installed. You can install - HpBandSter with the command: `pip install hpbandster` and - ConfigSpace with the command: `pip install ConfigSpace`. - - Parameters: - space (dict): Continuous ConfigSpace search space. Parameters will - be sampled from this space which will be used to run trials. - max_concurrent (int): Number of maximum concurrent trials. Defaults - to 10. - metric (str): The training result objective value attribute. - bohb_config (dict): configuration for HpBandSter BOHB algorithm - mode (str): One of {min, max}. Determines whether objective is - minimizing or maximizing the metric attribute. - - Example: - >>> CS = ConfigSpace - >>> config_space = CS.ConfigurationSpace() - >>> config_space.add_hyperparameter( - >>> CS.UniformFloatHyperparameter('width', lower=0, upper=20)) - >>> config_space.add_hyperparameter( - >>> CS.UniformFloatHyperparameter('height', lower=-100, upper=100)) - >>> config_space.add_hyperparameter( - >>> CS.CategoricalHyperparameter(name='activation', choices=['relu', 'tanh'])) - >>> current_best_params = [{ - >>> 'width': 10, - >>> 'height': 0, - >>> 'activation': 0, # The index of "relu" - >>> }] - >>> algo = TuneBOHB( - >>> config_space, max_concurrent=4, metric="mean_loss", mode="min") - """ + """BOHB suggestion component. + + Requires HpBandSter and ConfigSpace to be installed. You can install + HpBandSter with the command: `pip install hpbandster` and + ConfigSpace with the command: `pip install ConfigSpace`. + + This should be used in conjunction with HyperBandForBOHB. + + Parameters: + space (ConfigurationSpace): Continuous ConfigSpace search space. + Parameters will be sampled from this space which will be used + to run trials. + bohb_config (dict): configuration for HpBandSter BOHB algorithm + max_concurrent (int): Number of maximum concurrent trials. Defaults + to 10. + metric (str): The training result objective value attribute. + mode (str): One of {min, max}. Determines whether objective is + minimizing or maximizing the metric attribute. + + Example: + CS = ConfigSpace + config_space = CS.ConfigurationSpace() + config_space.add_hyperparameter( + CS.UniformFloatHyperparameter('width', lower=0, upper=20)) + config_space.add_hyperparameter( + CS.UniformFloatHyperparameter('height', lower=-100, upper=100)) + config_space.add_hyperparameter( + CS.CategoricalHyperparameter( + name='activation', choices=['relu', 'tanh'])) + algo = TuneBOHB( + config_space, max_concurrent=4, metric="mean_loss", mode="min") + bohb = HyperBandForBOHB( + time_attr="training_iteration", + metric="mean_loss", + mode="min", + max_t=100) + run(MyTrainableClass, scheduler=bohb, search_alg=algo) + + """ def __init__(self, space, + bohb_config=None, max_concurrent=10, metric="neg_mean_loss", - bohb_config=None, mode="max"): assert hpbandster is not None, "HpBandSter must be installed!" assert mode in ["min", "max"], "`mode` must be 'min' or 'max'!" @@ -91,6 +97,11 @@ def on_trial_result(self, trial_id, result): if "budget" in result.get("hyperband_info", {}): hbs_wrapper = self.to_wrapper(trial_id, result) self.bohber.new_result(hbs_wrapper) + else: + logger.warning( + "BOHB Info not detected in result. Are you using " + "HyperBandForBOHB as a scheduler?" + ) def on_trial_complete(self, trial_id, From cc0572678e810baed0fa1d8a14d5f73e8e9a4124 Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Sun, 11 Aug 2019 16:18:49 -0700 Subject: [PATCH 19/31] increase redundancy between HB and BOHB --- python/ray/tune/examples/bohb_example.py | 29 ++++++++++-------------- python/ray/tune/schedulers/hb_bohb.py | 12 +++------- python/ray/tune/schedulers/hyperband.py | 5 ++-- python/ray/tune/suggest/bohb.py | 7 +++--- 4 files changed, 21 insertions(+), 32 deletions(-) diff --git a/python/ray/tune/examples/bohb_example.py b/python/ray/tune/examples/bohb_example.py index aada30d65952..c1739b67d2aa 100644 --- a/python/ray/tune/examples/bohb_example.py +++ b/python/ray/tune/examples/bohb_example.py @@ -77,25 +77,20 @@ def _restore(self, checkpoint_path): max_t=100) config = { - "stop": { - "training_iteration": 1 if args.smoke_test else 99999 - }, - "num_samples": 20, - "resources_per_trial": { - "cpu": 1, - "gpu": 0 - }, - } + "stop": { + "training_iteration": 1 if args.smoke_test else 99999 + }, + "num_samples": 20, + "resources_per_trial": { + "cpu": 1, + "gpu": 0 + }, + } algo = TuneBOHB( - config_space, - max_concurrent=4, - metric="mean_loss", - mode="min" - ) - - run( - MyTrainableClass, + config_space, max_concurrent=4, metric="mean_loss", mode="min") + + run(MyTrainableClass, name="bohb_test", scheduler=bohb_hyperband, search_alg=algo, diff --git a/python/ray/tune/schedulers/hb_bohb.py b/python/ray/tune/schedulers/hb_bohb.py index 3871a91c93d3..de73b3573bf5 100644 --- a/python/ray/tune/schedulers/hb_bohb.py +++ b/python/ray/tune/schedulers/hb_bohb.py @@ -2,13 +2,11 @@ from __future__ import division from __future__ import print_function -import numpy as np import logging from ray.tune.schedulers.trial_scheduler import TrialScheduler from ray.tune.schedulers.hyperband import HyperBandScheduler, Bracket from ray.tune.trial import Trial -from ray.tune.error import TuneError logger = logging.getLogger(__name__) @@ -55,13 +53,9 @@ def on_trial_add(self, trial_runner, trial): cur_bracket = None else: retry = False - cur_bracket = Bracket( - self._time_attr, - self._get_n0(s), - self._get_r0(s), - self._max_t_attr, - self._eta, - s) + cur_bracket = Bracket(self._time_attr, self._get_n0(s), + self._get_r0(s), self._max_t_attr, + self._eta, s) cur_band.append(cur_bracket) self._state["bracket"] = cur_bracket diff --git a/python/ray/tune/schedulers/hyperband.py b/python/ray/tune/schedulers/hyperband.py index 6bd682cedb6c..d763f2ff5e1c 100644 --- a/python/ray/tune/schedulers/hyperband.py +++ b/python/ray/tune/schedulers/hyperband.py @@ -8,6 +8,7 @@ from ray.tune.schedulers.trial_scheduler import FIFOScheduler, TrialScheduler from ray.tune.trial import Trial +from ray.tune.error import TuneError logger = logging.getLogger(__name__) @@ -202,12 +203,12 @@ def _process_bracket(self, trial_runner, bracket, trial): bracket.cleanup_trial(t) action = TrialScheduler.STOP else: - raise Exception("Trial with unexpected status encountered") + raise TuneError("Trial with unexpected status encountered") # ready the good trials - if trial is too far ahead, don't continue for t in good: if t.status not in [Trial.PAUSED, Trial.RUNNING]: - raise Exception("Trial with unexpected status encountered") + raise TuneError("Trial with unexpected status encountered") if bracket.continue_trial(t): if t.status == Trial.PAUSED: self._unpause_trial(trial_runner, t) diff --git a/python/ray/tune/suggest/bohb.py b/python/ray/tune/suggest/bohb.py index d2ca83a03769..99b5977c4307 100644 --- a/python/ray/tune/suggest/bohb.py +++ b/python/ray/tune/suggest/bohb.py @@ -11,6 +11,7 @@ from ray.tune.suggest import SuggestionAlgorithm +logger = logging.getLogger(__name__) class JobWrapper(): """Dummy class""" @@ -98,10 +99,8 @@ def on_trial_result(self, trial_id, result): hbs_wrapper = self.to_wrapper(trial_id, result) self.bohber.new_result(hbs_wrapper) else: - logger.warning( - "BOHB Info not detected in result. Are you using " - "HyperBandForBOHB as a scheduler?" - ) + logger.warning("BOHB Info not detected in result. Are you using " + "HyperBandForBOHB as a scheduler?") def on_trial_complete(self, trial_id, From e34973bda3c991d41259086c18e4d7229c2fbb22 Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Sun, 11 Aug 2019 17:14:30 -0700 Subject: [PATCH 20/31] formatting and better example --- doc/source/tune-schedulers.rst | 17 +++++- doc/source/tune-searchalg.rst | 36 +++++++++++-- python/ray/tune/examples/bohb_example.py | 52 ++++++------------- python/ray/tune/schedulers/hyperband.py | 15 ++++-- python/ray/tune/suggest/bohb.py | 2 + python/ray/tune/tests/test_trial_scheduler.py | 44 +++++++++++++++- 6 files changed, 121 insertions(+), 45 deletions(-) diff --git a/doc/source/tune-schedulers.rst b/doc/source/tune-schedulers.rst index 12b802d06e8d..88ed64451da5 100644 --- a/doc/source/tune-schedulers.rst +++ b/doc/source/tune-schedulers.rst @@ -131,13 +131,28 @@ On the other hand, holding ``R`` constant at ``R = 300`` and varying ``eta`` als The implementation takes the same configuration as the example given in the paper and exposes ``max_t``, which is not a parameter in the paper. -2. The example in the `post `_ to calculate ``n_0`` is actually a little different than the algorithm given in the paper. In this implementation, we implement ``n_0`` according to the paper (which is `n` in the below example): +2. The example in the `post `_ to calculate ``n_0`` is actually a little different than the algorithm given in the paper. In this implementation, we implement ``n_0`` according to the paper (which is `n` in the below example): .. image:: images/hyperband_allocation.png 3. There are also implementation specific details like how trials are placed into brackets which are not covered in the paper. This implementation places trials within brackets according to smaller bracket first - meaning that with low number of trials, there will be less early stopping. +HyperBand (BOHB) +---------------- + +.. tip:: This implementation is still experimental. Please report issues on https://github.com/ray-project/ray/issues/. Thanks! + +This class is a variant of HyperBand that enables the BOHB Algorithm. This implementation is true to the original HyperBand implementation and does not implement pipelining nor straggler mitigation. + +This is to be used in conjunction with te Tune BOHB search algorithm. See `TuneBOHB `_ for package requirements, examples, and details. + +An example of this in use can be found in `bohb_example.py `__. + +.. autoclass:: ray.tune.schedulers.HyperBandForBOHB + :noindex: + + Median Stopping Rule -------------------- diff --git a/doc/source/tune-searchalg.rst b/doc/source/tune-searchalg.rst index 8fca40cc4650..35fb0852ef56 100644 --- a/doc/source/tune-searchalg.rst +++ b/doc/source/tune-searchalg.rst @@ -185,22 +185,48 @@ An example of this can be found in `ax_example.py `__ to perform sequential model-based hyperparameter optimization in conjunction with HyperBand. Note that this class does not extend ``ray.tune.suggest.BasicVariantGenerator``, so you will not be able to use Tune's default variant generation/search space declaration when using BOHB. Also note that BOHB is intended to be paired with a specific Hyperband scheduler `ray.tune.schedulers.HyperBandForBOHB` `__. +.. tip:: This implementation is still experimental. Please report issues on https://github.com/ray-project/ray/issues/. Thanks! -In order to use this search algorithm, you will need to install HpBandSter and ConfigSpace via the following command: +``BOHB`` (Bayesian Optimization HyperBand) is a SearchAlgorithm that is backed by `HpBandSter `__ to perform sequential model-based hyperparameter optimization in conjunction with HyperBand. Note that this class does not extend ``ray.tune.suggest.BasicVariantGenerator``, so you will not be able to use Tune's default variant generation/search space declaration when using BOHB. + +Importantly, BOHB is intended to be paired with a specific scheduler class: `HyperBandForBOHB <>`__. + +In order to use this search algorithm, you will need to install ``HpBandSter`` and ``ConfigSpace`` via the following command: .. code-block:: bash $ pip install hpbandster $ pip install ConfigSpace -This algorithm requires using the `ConfigSpace search space specification `__. You can use TuneBOHB like follows: +This algorithm requires using the `ConfigSpace search space specification `__. + + +You can use ``TuneBOHB`` in conjunction with ``HyperBandForBOHB`` as follows: .. code-block:: python - tune.run(... , search_alg=TuneBOHB(config_space, ... )) + # BOHB uses ConfigSpace for their hyperparameter search space + import ConfigSpace as CS + + config_space = CS.ConfigurationSpace() + config_space.add_hyperparameter( + CS.UniformFloatHyperparameter("height", lower=10, upper=100)) + config_space.add_hyperparameter( + CS.UniformFloatHyperparameter("width", lower=0, upper=100)) + + experiment_metrics = dict(metric="episode_reward_mean", mode="min") + bohb_hyperband = HyperBandForBOHB( + time_attr="training_iteration", max_t=100, **experiment_metrics) + bohb_search = TuneBOHB( + config_space, max_concurrent=4, **experiment_metrics) + + tune.run(MyTrainableClass, + name="bohb_test", + scheduler=bohb_hyperband, + search_alg=bohb_search, + num_samples=5) -An example of this can be found in `bohb_example.py `__. +Take a look at `an example here `_. See the `BOHB paper `_ for more details. .. autoclass:: ray.tune.suggest.bohb.TuneBOHB :show-inheritance: diff --git a/python/ray/tune/examples/bohb_example.py b/python/ray/tune/examples/bohb_example.py index c1739b67d2aa..888939b8cd98 100644 --- a/python/ray/tune/examples/bohb_example.py +++ b/python/ray/tune/examples/bohb_example.py @@ -4,6 +4,7 @@ from __future__ import division from __future__ import print_function +import argparse import json import os @@ -14,6 +15,14 @@ from ray.tune.schedulers.hb_bohb import HyperBandForBOHB from ray.tune.suggest.bohb import TuneBOHB +parser = argparse.ArgumentParser() +parser.add_argument( + "--smoke-test", action="store_true", help="Finish quickly for testing") +parser.add_argument( + "--ray-redis-address", + help="Address of Ray cluster for seamless distributed execution.") +args, _ = parser.parse_known_args() + class MyTrainableClass(Trainable): """Example agent whose learning curve is a random sigmoid. @@ -46,52 +55,25 @@ def _restore(self, checkpoint_path): if __name__ == "__main__": - import argparse - import ConfigSpace - - parser = argparse.ArgumentParser() - parser.add_argument( - "--smoke-test", action="store_true", help="Finish quickly for testing") - parser.add_argument( - "--ray-redis-address", - help="Address of Ray cluster for seamless distributed execution.") - args, _ = parser.parse_known_args() + import ConfigSpace as CS ray.init(redis_address=args.ray_redis_address) - # asynchronous hyperband early stopping, configured with - # `episode_reward_mean` as the - # objective and `training_iteration` as the time unit, - # which is automatically filled by Tune. - # BOHB uses ConfigSpace for their hyperparameter search space - CS = ConfigSpace config_space = CS.ConfigurationSpace() config_space.add_hyperparameter( CS.UniformFloatHyperparameter("height", lower=10, upper=100)) config_space.add_hyperparameter( CS.UniformFloatHyperparameter("width", lower=0, upper=100)) + experiment_metrics = dict(metric="episode_reward_mean", mode="min") bohb_hyperband = HyperBandForBOHB( - time_attr="training_iteration", - metric="episode_reward_mean", - max_t=100) - - config = { - "stop": { - "training_iteration": 1 if args.smoke_test else 99999 - }, - "num_samples": 20, - "resources_per_trial": { - "cpu": 1, - "gpu": 0 - }, - } - - algo = TuneBOHB( - config_space, max_concurrent=4, metric="mean_loss", mode="min") + time_attr="training_iteration", max_t=100, **experiment_metrics) + bohb_search = TuneBOHB( + config_space, max_concurrent=4, **experiment_metrics) run(MyTrainableClass, name="bohb_test", scheduler=bohb_hyperband, - search_alg=algo, - **config) + search_alg=bohb_search, + num_samples=5, + stop={"training_iteration": 10 if args.smoke_test else 100}) diff --git a/python/ray/tune/schedulers/hyperband.py b/python/ray/tune/schedulers/hyperband.py index d763f2ff5e1c..bdc8f9376401 100644 --- a/python/ray/tune/schedulers/hyperband.py +++ b/python/ray/tune/schedulers/hyperband.py @@ -73,6 +73,8 @@ class HyperBandScheduler(FIFOScheduler): The scheduler will terminate trials after this time has passed. Note that this is different from the semantics of `max_t` as mentioned in the original HyperBand paper. + reduction_factor (float): Same as `eta`. Determines how sharp + the difference is between bracket space-time allocation ratios. """ def __init__(self, @@ -80,7 +82,8 @@ def __init__(self, reward_attr=None, metric="episode_reward_mean", mode="max", - max_t=81): + max_t=81, + reduction_factor=3): assert max_t > 0, "Max (time_attr) not valid!" assert mode in ["min", "max"], "`mode` must be 'min' or 'max'!" @@ -93,8 +96,8 @@ def __init__(self, "Setting `metric={}` and `mode=max`.".format(reward_attr)) FIFOScheduler.__init__(self) - self._eta = 3 - self._s_max_1 = 5 + self._eta = reduction_factor + self._s_max_1 = int(np.log(max_t) / np.log(reduction_factor)) + 1 self._max_t_attr = max_t # bracket max trials self._get_n0 = lambda s: int( @@ -280,6 +283,12 @@ def debug_string(self): out += "\n {}".format(bracket) return out + def state(self): + return { + "num_brackets": sum(len(band) for band in self._hyperbands), + "num_stopped": self._num_stopped + } + def _unpause_trial(self, trial_runner, trial): trial_runner.trial_executor.unpause_trial(trial) diff --git a/python/ray/tune/suggest/bohb.py b/python/ray/tune/suggest/bohb.py index 99b5977c4307..962ea8cd76fb 100644 --- a/python/ray/tune/suggest/bohb.py +++ b/python/ray/tune/suggest/bohb.py @@ -4,6 +4,7 @@ from __future__ import division from __future__ import print_function +import logging try: from hpbandster import BOHB except ImportError: @@ -13,6 +14,7 @@ logger = logging.getLogger(__name__) + class JobWrapper(): """Dummy class""" diff --git a/python/ray/tune/tests/test_trial_scheduler.py b/python/ray/tune/tests/test_trial_scheduler.py index 2a0282c5a975..2f23bba8af09 100644 --- a/python/ray/tune/tests/test_trial_scheduler.py +++ b/python/ray/tune/tests/test_trial_scheduler.py @@ -15,7 +15,7 @@ from ray.tune.result import TRAINING_ITERATION from ray.tune.schedulers import (HyperBandScheduler, AsyncHyperBandScheduler, PopulationBasedTraining, MedianStoppingRule, - TrialScheduler) + TrialScheduler, HyperBandForBOHB) from ray.tune.schedulers.pbt import explore from ray.tune.trial import Trial, Checkpoint @@ -603,6 +603,48 @@ def testFilterNoneBracket(self): self.assertIsNotNone(trial) +class BOHBSuite(unittest.TestCase): + def setUp(self): + ray.init() + + def tearDown(self): + ray.shutdown() + _register_all() # re-register the evicted objects + + def testLargestBracketFirst(self): + sched = HyperBandForBOHB(max_t=3, reduction_factor=3) + runner = _MockTrialRunner(sched) + for i in range(3): + sched.on_trial_add(runner, Trial("__fake")) + + self.assertEqual(sched.state()["num_brackets"], 1) + sched.on_trial_add(runner, Trial("__fake")) + self.assertEqual(sched.state()["num_brackets"], 2) + + def testCheckTrialInfoUpdate(self): + def result(score, ts): + return {"episode_reward_mean": score, TRAINING_ITERATION: ts} + + sched = HyperBandScheduler(max_t=3, reduction_factor=3) + runner = _MockTrialRunner(sched) + runner._search_alg = MagicMock() + trials = [Trial("__fake") for i in range(3)] + for t in trials: + sched.on_trial_add(runner, t) + + decision = sched.on_trial_result(runner, trials[0], result(1, 1)) + self.assertEqual(decision, TrialScheduler.PAUSE) + decision = sched.on_trial_result(runner, trials[1], result(1, 1)) + self.assertEqual(decision, TrialScheduler.PAUSE) + decision = sched.on_trial_result(runner, trials[2], result(1, 1)) + self.assertEqual(decision, TrialScheduler.CONTINUE) + + self.assertEqual(runner._search_alg.on_pause.call_count, 3) + self.assertEqual(runner._search_alg.on_unpause.call_count, 1) + call_args = runner._search_alg.on_trial_result.call_args + self.assertTrue("hyperband_info" in call_args[2]) + + class _MockTrial(Trial): def __init__(self, i, config): self.trainable_name = "trial_{}".format(i) From 09daf859a59a843be1c502e2d7ba0c8f0a203bbd Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Sun, 11 Aug 2019 17:22:17 -0700 Subject: [PATCH 21/31] docs --- doc/source/tune-schedulers.rst | 4 ++-- python/ray/tune/schedulers/hb_bohb.py | 10 ++++++---- python/ray/tune/suggest/bohb.py | 7 ++++--- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/doc/source/tune-schedulers.rst b/doc/source/tune-schedulers.rst index 88ed64451da5..62938c09628f 100644 --- a/doc/source/tune-schedulers.rst +++ b/doc/source/tune-schedulers.rst @@ -145,9 +145,9 @@ HyperBand (BOHB) This class is a variant of HyperBand that enables the BOHB Algorithm. This implementation is true to the original HyperBand implementation and does not implement pipelining nor straggler mitigation. -This is to be used in conjunction with te Tune BOHB search algorithm. See `TuneBOHB `_ for package requirements, examples, and details. +This is to be used in conjunction with the Tune BOHB search algorithm. See `TuneBOHB `_ for package requirements, examples, and details. -An example of this in use can be found in `bohb_example.py `__. +An example of this in use can be found in `bohb_example.py `_. .. autoclass:: ray.tune.schedulers.HyperBandForBOHB :noindex: diff --git a/python/ray/tune/schedulers/hb_bohb.py b/python/ray/tune/schedulers/hb_bohb.py index de73b3573bf5..ecb954d440de 100644 --- a/python/ray/tune/schedulers/hb_bohb.py +++ b/python/ray/tune/schedulers/hb_bohb.py @@ -16,10 +16,12 @@ class HyperBandForBOHB(HyperBandScheduler): The default HyperBandScheduler implements pipelining for efficiency. This class introduces key changes: - - Trials are now placed so that the bracket with the largest size - is filled first. - - Trials will be paused even if the bracket is not filled. This allows - BOHB to insert new trials into the training. + + 1. Trials are now placed so that the bracket with the largest size is + filled first. + + 2.Trials will be paused even if the bracket is not filled. This allows + BOHB to insert new trials into the training. """ diff --git a/python/ray/tune/suggest/bohb.py b/python/ray/tune/suggest/bohb.py index 962ea8cd76fb..5795d3fbe9a9 100644 --- a/python/ray/tune/suggest/bohb.py +++ b/python/ray/tune/suggest/bohb.py @@ -27,9 +27,10 @@ def __init__(self, loss, budget, config): class TuneBOHB(SuggestionAlgorithm): """BOHB suggestion component. - Requires HpBandSter and ConfigSpace to be installed. You can install - HpBandSter with the command: `pip install hpbandster` and - ConfigSpace with the command: `pip install ConfigSpace`. + Requires HpBandSter and ConfigSpace to be installed. + You can install HpBandSter and ConfigSpace with: + + `pip install hpbandster ConfigSpace` This should be used in conjunction with HyperBandForBOHB. From b08054962d68bf71129387fd001366cbfe553be7 Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Sun, 11 Aug 2019 23:43:53 -0700 Subject: [PATCH 22/31] better tests --- doc/source/tune-searchalg.rst | 9 ++-- python/ray/tune/schedulers/hb_bohb.py | 7 ++- python/ray/tune/schedulers/hyperband.py | 9 ++-- python/ray/tune/suggest/bohb.py | 43 +++++++-------- python/ray/tune/tests/test_trial_scheduler.py | 53 ++++++++++--------- 5 files changed, 62 insertions(+), 59 deletions(-) diff --git a/doc/source/tune-searchalg.rst b/doc/source/tune-searchalg.rst index 35fb0852ef56..fc452630761d 100644 --- a/doc/source/tune-searchalg.rst +++ b/doc/source/tune-searchalg.rst @@ -189,16 +189,13 @@ BOHB ``BOHB`` (Bayesian Optimization HyperBand) is a SearchAlgorithm that is backed by `HpBandSter `__ to perform sequential model-based hyperparameter optimization in conjunction with HyperBand. Note that this class does not extend ``ray.tune.suggest.BasicVariantGenerator``, so you will not be able to use Tune's default variant generation/search space declaration when using BOHB. -Importantly, BOHB is intended to be paired with a specific scheduler class: `HyperBandForBOHB <>`__. +Importantly, BOHB is intended to be paired with a specific scheduler class: `HyperBandForBOHB `__. -In order to use this search algorithm, you will need to install ``HpBandSter`` and ``ConfigSpace`` via the following command: +This algorithm requires using the `ConfigSpace search space specification `_. In order to use this search algorithm, you will need to install ``HpBandSter`` and ``ConfigSpace``: .. code-block:: bash - $ pip install hpbandster - $ pip install ConfigSpace - -This algorithm requires using the `ConfigSpace search space specification `__. + $ pip install hpbandster ConfigSpace You can use ``TuneBOHB`` in conjunction with ``HyperBandForBOHB`` as follows: diff --git a/python/ray/tune/schedulers/hb_bohb.py b/python/ray/tune/schedulers/hb_bohb.py index ecb954d440de..d76a63f1f4be 100644 --- a/python/ray/tune/schedulers/hb_bohb.py +++ b/python/ray/tune/schedulers/hb_bohb.py @@ -23,6 +23,7 @@ class introduces key changes: 2.Trials will be paused even if the bracket is not filled. This allows BOHB to insert new trials into the training. + See ray.tune.schedulers.HyperBandScheduler for parameter docstring. """ def on_trial_add(self, trial_runner, trial): @@ -90,7 +91,7 @@ def on_trial_result(self, trial_runner, trial, result): if t is not trial): trial_runner._search_alg.on_pause(trial.trial_id) return TrialScheduler.PAUSE - action = self._process_bracket(trial_runner, bracket, trial) + action = self._process_bracket(trial_runner, bracket) return action def _unpause_trial(self, trial_runner, trial): @@ -121,6 +122,8 @@ def choose_trial_to_run(self, trial_runner): for bracket in hyperband: if bracket and any(trial.status == Trial.PAUSED for trial in bracket.current_trials()): - self._process_bracket(trial_runner, bracket, None) + # This will change the trial state and let the + # trial runner retry. + self._process_bracket(trial_runner, bracket) # MAIN CHANGE HERE! return None diff --git a/python/ray/tune/schedulers/hyperband.py b/python/ray/tune/schedulers/hyperband.py index bdc8f9376401..064ae09aa8bd 100644 --- a/python/ray/tune/schedulers/hyperband.py +++ b/python/ray/tune/schedulers/hyperband.py @@ -97,7 +97,8 @@ def __init__(self, FIFOScheduler.__init__(self) self._eta = reduction_factor - self._s_max_1 = int(np.log(max_t) / np.log(reduction_factor)) + 1 + self._s_max_1 = int( + np.round(np.log(max_t) / np.log(reduction_factor))) + 1 self._max_t_attr = max_t # bracket max trials self._get_n0 = lambda s: int( @@ -177,10 +178,10 @@ def on_trial_result(self, trial_runner, trial, result): if bracket.continue_trial(trial): return TrialScheduler.CONTINUE - action = self._process_bracket(trial_runner, bracket, trial) + action = self._process_bracket(trial_runner, bracket) return action - def _process_bracket(self, trial_runner, bracket, trial): + def _process_bracket(self, trial_runner, bracket): """This is called whenever a trial makes progress. When all live trials in the bracket have no more iterations left, @@ -227,7 +228,7 @@ def on_trial_remove(self, trial_runner, trial): bracket, _ = self._trial_info[trial] bracket.cleanup_trial(trial) if not bracket.finished(): - self._process_bracket(trial_runner, bracket, trial) + self._process_bracket(trial_runner, bracket) def on_trial_complete(self, trial_runner, trial, result): """Cleans up trial info from bracket if trial completed early.""" diff --git a/python/ray/tune/suggest/bohb.py b/python/ray/tune/suggest/bohb.py index 5795d3fbe9a9..21716f134c1b 100644 --- a/python/ray/tune/suggest/bohb.py +++ b/python/ray/tune/suggest/bohb.py @@ -16,8 +16,6 @@ class JobWrapper(): - """Dummy class""" - def __init__(self, loss, budget, config): self.result = {"loss": loss} self.kwargs = {"budget": budget, "config": config.copy()} @@ -27,14 +25,13 @@ def __init__(self, loss, budget, config): class TuneBOHB(SuggestionAlgorithm): """BOHB suggestion component. - Requires HpBandSter and ConfigSpace to be installed. - You can install HpBandSter and ConfigSpace with: - `pip install hpbandster ConfigSpace` + Requires HpBandSter and ConfigSpace to be installed. You can install + HpBandSter and ConfigSpace with: `pip install hpbandster ConfigSpace`. This should be used in conjunction with HyperBandForBOHB. - Parameters: + Args: space (ConfigurationSpace): Continuous ConfigSpace search space. Parameters will be sampled from this space which will be used to run trials. @@ -46,23 +43,23 @@ class TuneBOHB(SuggestionAlgorithm): minimizing or maximizing the metric attribute. Example: - CS = ConfigSpace - config_space = CS.ConfigurationSpace() - config_space.add_hyperparameter( - CS.UniformFloatHyperparameter('width', lower=0, upper=20)) - config_space.add_hyperparameter( - CS.UniformFloatHyperparameter('height', lower=-100, upper=100)) - config_space.add_hyperparameter( - CS.CategoricalHyperparameter( - name='activation', choices=['relu', 'tanh'])) - algo = TuneBOHB( - config_space, max_concurrent=4, metric="mean_loss", mode="min") - bohb = HyperBandForBOHB( - time_attr="training_iteration", - metric="mean_loss", - mode="min", - max_t=100) - run(MyTrainableClass, scheduler=bohb, search_alg=algo) + >>> import ConfigSpace as CS + >>> config_space = CS.ConfigurationSpace() + >>> config_space.add_hyperparameter( + CS.UniformFloatHyperparameter('width', lower=0, upper=20)) + >>> config_space.add_hyperparameter( + CS.UniformFloatHyperparameter('height', lower=-100, upper=100)) + >>> config_space.add_hyperparameter( + CS.CategoricalHyperparameter( + name='activation', choices=['relu', 'tanh'])) + >>> algo = TuneBOHB( + config_space, max_concurrent=4, metric='mean_loss', mode='min') + >>> bohb = HyperBandForBOHB( + time_attr='training_iteration', + metric='mean_loss', + mode='min', + max_t=100) + >>> run(MyTrainableClass, scheduler=bohb, search_alg=algo) """ diff --git a/python/ray/tune/tests/test_trial_scheduler.py b/python/ray/tune/tests/test_trial_scheduler.py index 2f23bba8af09..d1f9d0ac8402 100644 --- a/python/ray/tune/tests/test_trial_scheduler.py +++ b/python/ray/tune/tests/test_trial_scheduler.py @@ -236,7 +236,7 @@ def _launch_trial(self, trial): class HyperbandSuite(unittest.TestCase): def setUp(self): - ray.init() + ray.init(object_store_memory=int(1e8)) def tearDown(self): ray.shutdown() @@ -319,17 +319,19 @@ def testConfigSameEta(self): self.assertEqual(sched._hyperbands[0][-1]._n, 81) self.assertEqual(sched._hyperbands[0][-1]._r, 1) - sched = HyperBandScheduler(max_t=810) + reduction_factor = 10 + sched = HyperBandScheduler( + max_t=1000, reduction_factor=reduction_factor) i = 0 while not sched._cur_band_filled(): t = Trial("__fake") sched.on_trial_add(None, t) i += 1 - self.assertEqual(len(sched._hyperbands[0]), 5) - self.assertEqual(sched._hyperbands[0][0]._n, 5) - self.assertEqual(sched._hyperbands[0][0]._r, 810) - self.assertEqual(sched._hyperbands[0][-1]._n, 81) - self.assertEqual(sched._hyperbands[0][-1]._r, 10) + self.assertEqual(len(sched._hyperbands[0]), 4) + self.assertEqual(sched._hyperbands[0][0]._n, 4) + self.assertEqual(sched._hyperbands[0][0]._r, 1000) + self.assertEqual(sched._hyperbands[0][-1]._n, 1000) + self.assertEqual(sched._hyperbands[0][-1]._r, 1) def testConfigSameEtaSmall(self): sched = HyperBandScheduler(max_t=1) @@ -338,8 +340,7 @@ def testConfigSameEtaSmall(self): t = Trial("__fake") sched.on_trial_add(None, t) i += 1 - self.assertEqual(len(sched._hyperbands[0]), 5) - self.assertTrue(all(v is None for v in sched._hyperbands[0][1:])) + self.assertEqual(len(sched._hyperbands[0]), 1) def testSuccessiveHalving(self): """Setup full band, then iterate through last bracket (n=81) @@ -605,7 +606,7 @@ def testFilterNoneBracket(self): class BOHBSuite(unittest.TestCase): def setUp(self): - ray.init() + ray.init(object_store_memory=int(1e8)) def tearDown(self): ray.shutdown() @@ -615,7 +616,9 @@ def testLargestBracketFirst(self): sched = HyperBandForBOHB(max_t=3, reduction_factor=3) runner = _MockTrialRunner(sched) for i in range(3): - sched.on_trial_add(runner, Trial("__fake")) + t = Trial("__fake") + sched.on_trial_add(runner, t) + runner._launch_trial(t) self.assertEqual(sched.state()["num_brackets"], 1) sched.on_trial_add(runner, Trial("__fake")) @@ -625,24 +628,26 @@ def testCheckTrialInfoUpdate(self): def result(score, ts): return {"episode_reward_mean": score, TRAINING_ITERATION: ts} - sched = HyperBandScheduler(max_t=3, reduction_factor=3) + sched = HyperBandForBOHB(max_t=3, reduction_factor=3) runner = _MockTrialRunner(sched) runner._search_alg = MagicMock() trials = [Trial("__fake") for i in range(3)] for t in trials: - sched.on_trial_add(runner, t) - - decision = sched.on_trial_result(runner, trials[0], result(1, 1)) - self.assertEqual(decision, TrialScheduler.PAUSE) - decision = sched.on_trial_result(runner, trials[1], result(1, 1)) - self.assertEqual(decision, TrialScheduler.PAUSE) - decision = sched.on_trial_result(runner, trials[2], result(1, 1)) - self.assertEqual(decision, TrialScheduler.CONTINUE) - - self.assertEqual(runner._search_alg.on_pause.call_count, 3) + runner.add_trial(t) + runner._launch_trial(t) + + for trial, trial_result in zip(trials, [result(1, 1), result(2, 1)]): + decision = sched.on_trial_result(runner, trial, trial_result) + self.assertEqual(decision, TrialScheduler.PAUSE) + runner._pause_trial(trial) + spy_result = result(1, 1) + decision = sched.on_trial_result(runner, trials[-1], spy_result) + self.assertEqual(decision, TrialScheduler.STOP) + sched.choose_trial_to_run(runner) + self.assertEqual(runner._search_alg.on_pause.call_count, 2) self.assertEqual(runner._search_alg.on_unpause.call_count, 1) - call_args = runner._search_alg.on_trial_result.call_args - self.assertTrue("hyperband_info" in call_args[2]) + self.assertTrue("hyperband_info" in spy_result) + self.assertEquals(spy_result["hyperband_info"]["budget"], 1) class _MockTrial(Trial): From 27f941a240afaeb906b69cef6bafac855a86976d Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Sun, 11 Aug 2019 23:47:30 -0700 Subject: [PATCH 23/31] suggest --- python/ray/tune/suggest/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/ray/tune/suggest/__init__.py b/python/ray/tune/suggest/__init__.py index 69f2897207ae..a182f6b1ae43 100644 --- a/python/ray/tune/suggest/__init__.py +++ b/python/ray/tune/suggest/__init__.py @@ -2,10 +2,11 @@ from ray.tune.suggest.basic_variant import BasicVariantGenerator from ray.tune.suggest.suggestion import SuggestionAlgorithm from ray.tune.suggest.variant_generator import grid_search +from ray.tune.suggest.bohb import TuneBOHB __all__ = [ "SearchAlgorithm", "BasicVariantGenerator", "SuggestionAlgorithm", - "grid_search" + "grid_search", "TuneBOHB" ] From 3d42a8b5950060324a1101f8c2fbef4f2f92f82b Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Sun, 11 Aug 2019 23:54:07 -0700 Subject: [PATCH 24/31] dogstrings --- python/ray/tune/schedulers/hb_bohb.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/python/ray/tune/schedulers/hb_bohb.py b/python/ray/tune/schedulers/hb_bohb.py index d76a63f1f4be..428ffa6f7b08 100644 --- a/python/ray/tune/schedulers/hb_bohb.py +++ b/python/ray/tune/schedulers/hb_bohb.py @@ -14,13 +14,13 @@ class HyperBandForBOHB(HyperBandScheduler): """Extends HyperBand early stopping algorithm for BOHB. - The default HyperBandScheduler implements pipelining for efficiency. This + This implementation removes the ``HyperBandScheduler`` pipelining. This class introduces key changes: 1. Trials are now placed so that the bracket with the largest size is filled first. - 2.Trials will be paused even if the bracket is not filled. This allows + 2. Trials will be paused even if the bracket is not filled. This allows BOHB to insert new trials into the training. See ray.tune.schedulers.HyperBandScheduler for parameter docstring. @@ -29,10 +29,10 @@ class introduces key changes: def on_trial_add(self, trial_runner, trial): """Adds new trial. - On a new trial add, if current bracket is not filled, - add to current bracket. Else, if current band is not filled, - create new bracket, add to current bracket. - Else, create new iteration, create new bracket, add to bracket. + On a new trial add, if current bracket is not filled, add to current + bracket. Else, if current band is not filled, create new bracket, add + to current bracket. Else, create new iteration, create new bracket, + add to bracket. """ cur_bracket = self._state["bracket"] @@ -50,9 +50,8 @@ def on_trial_add(self, trial_runner, trial): # cur_band will always be less than s_max_1 or else filled s = self._s_max_1 - len(cur_band) - 1 assert s >= 0, "Current band is filled!" - # MAIN CHANGE HERE! if self._get_r0(s) == 0: - logger.info("Bracket too small - Retrying...") + logger.debug("BOHB: Bracket too small - Retrying...") cur_bracket = None else: retry = False From ce0c1c77b7e22435697f85f2614b141c9a774675 Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Sun, 11 Aug 2019 23:57:58 -0700 Subject: [PATCH 25/31] ok reverse --- python/ray/tune/tests/test_trial_scheduler.py | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/python/ray/tune/tests/test_trial_scheduler.py b/python/ray/tune/tests/test_trial_scheduler.py index d1f9d0ac8402..8f354d10a87e 100644 --- a/python/ray/tune/tests/test_trial_scheduler.py +++ b/python/ray/tune/tests/test_trial_scheduler.py @@ -640,7 +640,7 @@ def result(score, ts): decision = sched.on_trial_result(runner, trial, trial_result) self.assertEqual(decision, TrialScheduler.PAUSE) runner._pause_trial(trial) - spy_result = result(1, 1) + spy_result = result(0, 1) decision = sched.on_trial_result(runner, trials[-1], spy_result) self.assertEqual(decision, TrialScheduler.STOP) sched.choose_trial_to_run(runner) @@ -649,6 +649,30 @@ def result(score, ts): self.assertTrue("hyperband_info" in spy_result) self.assertEquals(spy_result["hyperband_info"]["budget"], 1) + def testCheckTrialInfoUpdateMin(self): + def result(score, ts): + return {"episode_reward_mean": score, TRAINING_ITERATION: ts} + + sched = HyperBandForBOHB(max_t=3, reduction_factor=3, mode="min") + runner = _MockTrialRunner(sched) + runner._search_alg = MagicMock() + trials = [Trial("__fake") for i in range(3)] + for t in trials: + runner.add_trial(t) + runner._launch_trial(t) + + for trial, trial_result in zip(trials, [result(1, 1), result(2, 1)]): + decision = sched.on_trial_result(runner, trial, trial_result) + self.assertEqual(decision, TrialScheduler.PAUSE) + runner._pause_trial(trial) + spy_result = result(0, 1) + decision = sched.on_trial_result(runner, trials[-1], spy_result) + self.assertEqual(decision, TrialScheduler.CONTINUE) + sched.choose_trial_to_run(runner) + self.assertEqual(runner._search_alg.on_pause.call_count, 2) + self.assertTrue("hyperband_info" in spy_result) + self.assertEquals(spy_result["hyperband_info"]["budget"], 1) + class _MockTrial(Trial): def __init__(self, i, config): From bb47d7de61a6fd5c11cec01d1b21303b46bff935 Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Mon, 12 Aug 2019 00:07:03 -0700 Subject: [PATCH 26/31] test --- python/ray/tune/suggest/bohb.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python/ray/tune/suggest/bohb.py b/python/ray/tune/suggest/bohb.py index 21716f134c1b..dae6dca1a7a3 100644 --- a/python/ray/tune/suggest/bohb.py +++ b/python/ray/tune/suggest/bohb.py @@ -15,7 +15,8 @@ logger = logging.getLogger(__name__) -class JobWrapper(): +class _BOHBJobWrapper(): + """Mock object for HpBandSter to process.""" def __init__(self, loss, budget, config): self.result = {"loss": loss} self.kwargs = {"budget": budget, "config": config.copy()} @@ -114,7 +115,7 @@ def on_trial_complete(self, self.running.remove(trial_id) def to_wrapper(self, trial_id, result): - return JobWrapper( + return _BOHBJobWrapper( self._metric_op * result[self.metric], result["hyperband_info"]["budget"], {k: result["config"][k] From 1c6e598ce90c1c81c918f589c19a6280418fc714 Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Mon, 12 Aug 2019 00:34:27 -0700 Subject: [PATCH 27/31] fixexample --- python/ray/tune/examples/bohb_example.py | 7 ++++-- python/ray/tune/suggest/bohb.py | 29 ++++++++++++------------ 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/python/ray/tune/examples/bohb_example.py b/python/ray/tune/examples/bohb_example.py index 888939b8cd98..11cac9790a0f 100644 --- a/python/ray/tune/examples/bohb_example.py +++ b/python/ray/tune/examples/bohb_example.py @@ -67,7 +67,10 @@ def _restore(self, checkpoint_path): experiment_metrics = dict(metric="episode_reward_mean", mode="min") bohb_hyperband = HyperBandForBOHB( - time_attr="training_iteration", max_t=100, **experiment_metrics) + time_attr="training_iteration", + max_t=100, + reduction_factor=4, + **experiment_metrics) bohb_search = TuneBOHB( config_space, max_concurrent=4, **experiment_metrics) @@ -75,5 +78,5 @@ def _restore(self, checkpoint_path): name="bohb_test", scheduler=bohb_hyperband, search_alg=bohb_search, - num_samples=5, + num_samples=10, stop={"training_iteration": 10 if args.smoke_test else 100}) diff --git a/python/ray/tune/suggest/bohb.py b/python/ray/tune/suggest/bohb.py index dae6dca1a7a3..18ebd43ea31d 100644 --- a/python/ray/tune/suggest/bohb.py +++ b/python/ray/tune/suggest/bohb.py @@ -4,11 +4,12 @@ from __future__ import division from __future__ import print_function +import copy import logging try: - from hpbandster import BOHB + from hpbandster.optimizers.config_generators.bohb import BOHB except ImportError: - hpbandster = None + BOHB = None from ray.tune.suggest import SuggestionAlgorithm @@ -17,6 +18,7 @@ class _BOHBJobWrapper(): """Mock object for HpBandSter to process.""" + def __init__(self, loss, budget, config): self.result = {"loss": loss} self.kwargs = {"budget": budget, "config": config.copy()} @@ -70,7 +72,7 @@ def __init__(self, max_concurrent=10, metric="neg_mean_loss", mode="max"): - assert hpbandster is not None, "HpBandSter must be installed!" + assert BOHB is not None, "HpBandSter must be installed!" assert mode in ["min", "max"], "`mode` must be 'min' or 'max'!" self._max_concurrent = max_concurrent self.trial_to_params = {} @@ -87,8 +89,9 @@ def __init__(self, def _suggest(self, trial_id): if len(self.running) < self._max_concurrent: - config, info = self.bohber.get_config() - self.trial_to_params[trial_id] = list(config) + # This parameter is not used in hpbandster implementation. + config, info = self.bohber.get_config(None) + self.trial_to_params[trial_id] = copy.deepcopy(config) self.running.add(trial_id) return config return None @@ -96,12 +99,12 @@ def _suggest(self, trial_id): def on_trial_result(self, trial_id, result): if trial_id not in self.paused: self.running.add(trial_id) - if "budget" in result.get("hyperband_info", {}): - hbs_wrapper = self.to_wrapper(trial_id, result) - self.bohber.new_result(hbs_wrapper) - else: + if "hyperband_info" not in result: logger.warning("BOHB Info not detected in result. Are you using " "HyperBandForBOHB as a scheduler?") + elif "budget" in result.get("hyperband_info", {}): + hbs_wrapper = self.to_wrapper(trial_id, result) + self.bohber.new_result(hbs_wrapper) def on_trial_complete(self, trial_id, @@ -115,11 +118,9 @@ def on_trial_complete(self, self.running.remove(trial_id) def to_wrapper(self, trial_id, result): - return _BOHBJobWrapper( - self._metric_op * result[self.metric], - result["hyperband_info"]["budget"], - {k: result["config"][k] - for k in self.trial_to_params[trial_id]}) + return _BOHBJobWrapper(self._metric_op * result[self.metric], + result["hyperband_info"]["budget"], + self.trial_to_params[trial_id]) def on_pause(self, trial_id): self.paused.add(trial_id) From 77b1d613818624915bfc30de012593a609f5f451 Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Mon, 12 Aug 2019 11:10:50 -0700 Subject: [PATCH 28/31] dockerfile? --- docker/examples/Dockerfile | 6 +----- docker/tune_test/Dockerfile | 4 +--- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/docker/examples/Dockerfile b/docker/examples/Dockerfile index 2b2e804bd7d9..9cbc3ae78b09 100644 --- a/docker/examples/Dockerfile +++ b/docker/examples/Dockerfile @@ -11,10 +11,6 @@ RUN pip install gym[atari] opencv-python-headless tensorflow lz4 keras pytest-ti RUN pip install -U h5py # Mutes FutureWarnings RUN pip install --upgrade bayesian-optimization RUN pip install --upgrade git+git://github.com/hyperopt/hyperopt.git -RUN pip install --upgrade sigopt -RUN pip install --upgrade nevergrad -RUN pip install --upgrade scikit-optimize -RUN pip install --upgrade hpbandster -RUN pip install --upgrade ConfigSpace +RUN pip install --upgrade sigopt nevergrad scikit-optimize hpbandster ConfigSpace RUN pip install -U pytest-remotedata>=0.3.1 RUN conda install pytorch-cpu torchvision-cpu -c pytorch diff --git a/docker/tune_test/Dockerfile b/docker/tune_test/Dockerfile index 75ae4e8d9025..f30e156a1cd2 100644 --- a/docker/tune_test/Dockerfile +++ b/docker/tune_test/Dockerfile @@ -13,9 +13,7 @@ RUN conda remove -y --force wrapt RUN pip install gym[atari]==0.10.11 opencv-python-headless tensorflow lz4 keras pytest-timeout smart_open RUN pip install --upgrade bayesian-optimization RUN pip install --upgrade git+git://github.com/hyperopt/hyperopt.git -RUN pip install --upgrade sigopt -RUN pip install --upgrade nevergrad -RUN pip install --upgrade scikit-optimize +RUN pip install --upgrade sigopt nevergrad scikit-optimize hpbandster ConfigSpace RUN pip install -U pytest-remotedata>=0.3.1 RUN conda install pytorch-cpu torchvision-cpu -c pytorch From 44e8dc65bbb3fb9cb1bd8c5018f833f9dfec6bb6 Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Mon, 12 Aug 2019 12:01:11 -0700 Subject: [PATCH 29/31] fix --- python/ray/tune/suggest/bohb.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/python/ray/tune/suggest/bohb.py b/python/ray/tune/suggest/bohb.py index 18ebd43ea31d..3fe5d5877f29 100644 --- a/python/ray/tune/suggest/bohb.py +++ b/python/ray/tune/suggest/bohb.py @@ -6,10 +6,6 @@ import copy import logging -try: - from hpbandster.optimizers.config_generators.bohb import BOHB -except ImportError: - BOHB = None from ray.tune.suggest import SuggestionAlgorithm @@ -72,6 +68,7 @@ def __init__(self, max_concurrent=10, metric="neg_mean_loss", mode="max"): + from hpbandster.optimizers.config_generators.bohb import BOHB assert BOHB is not None, "HpBandSter must be installed!" assert mode in ["min", "max"], "`mode` must be 'min' or 'max'!" self._max_concurrent = max_concurrent From 5fec52aa9faf0623b5222bd827d96ff8a3186199 Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Mon, 12 Aug 2019 14:59:34 -0700 Subject: [PATCH 30/31] fix --- ci/jenkins_tests/run_tune_tests.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ci/jenkins_tests/run_tune_tests.sh b/ci/jenkins_tests/run_tune_tests.sh index 31b7b4b0acc5..1b1cd5a8a8b9 100755 --- a/ci/jenkins_tests/run_tune_tests.sh +++ b/ci/jenkins_tests/run_tune_tests.sh @@ -110,6 +110,7 @@ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} python /ray/python/ray/tune/examples/skopt_example.py \ --smoke-test -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - python /ray/python/ray/tune/examples/bohb_example.py \ - --smoke-test +# uncomment once statsmodels is updated. +# $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ +# python /ray/python/ray/tune/examples/bohb_example.py \ +# --smoke-test From bbdaa01ddfe2c8d7afadc87955d5e78926becb57 Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Mon, 12 Aug 2019 23:26:15 -0700 Subject: [PATCH 31/31] bohb --- python/ray/tune/tests/test_trial_scheduler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ray/tune/tests/test_trial_scheduler.py b/python/ray/tune/tests/test_trial_scheduler.py index 8f354d10a87e..4d9eb8d072fa 100644 --- a/python/ray/tune/tests/test_trial_scheduler.py +++ b/python/ray/tune/tests/test_trial_scheduler.py @@ -368,7 +368,7 @@ def testSuccessiveHalving(self): self.assertEqual(action, TrialScheduler.CONTINUE) new_length = len(big_bracket.current_trials()) self.assertEqual(new_length, self.downscale(current_length, sched)) - cur_units += int(cur_units * sched._eta) + cur_units = int(cur_units * sched._eta) self.assertEqual(len(big_bracket.current_trials()), 1) def testHalvingStop(self):