From ff324dcd86b620407f263b45d595868a416fbf59 Mon Sep 17 00:00:00 2001 From: RaphaelCS Date: Fri, 26 Feb 2021 11:31:25 +0800 Subject: [PATCH 1/2] [tune] Correctly validate nested metrics Before: - Nested metrics couldn't pass validation process, since the nested result was used to validate metrics After: - Flattened result is used to validate metrics --- python/ray/tune/trial_runner.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/ray/tune/trial_runner.py b/python/ray/tune/trial_runner.py index 21b464f626e2..cf7c7d9edadb 100644 --- a/python/ray/tune/trial_runner.py +++ b/python/ray/tune/trial_runner.py @@ -739,10 +739,11 @@ def _process_trial_result(self, trial, result): result = trial.last_result result.update(done=True) - self._validate_result_metrics(result) self._total_time += result.get(TIME_THIS_ITER_S, 0) flat_result = flatten_dict(result) + self._validate_result_metrics(flat_result) + if self._stopper(trial.trial_id, result) or trial.should_stop(flat_result): result.update(done=True) From e5f6f35d380c0aeb0e3a4cb53868a883939eb869 Mon Sep 17 00:00:00 2001 From: Kai Fricke Date: Fri, 26 Feb 2021 11:31:54 +0100 Subject: [PATCH 2/2] Fix BO test and lint --- python/ray/tune/tests/test_sample.py | 2 +- python/ray/tune/trial_runner.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/ray/tune/tests/test_sample.py b/python/ray/tune/tests/test_sample.py index 311b7b397c51..4ca8f0d003fe 100644 --- a/python/ray/tune/tests/test_sample.py +++ b/python/ray/tune/tests/test_sample.py @@ -342,7 +342,7 @@ def testConvertBayesOpt(self): with self.assertRaises(ValueError): searcher.set_search_properties("none", "max", invalid_config) - searcher = BayesOptSearch(metric="b", mode="max") + searcher = BayesOptSearch(metric="b/z", mode="max") analysis = tune.run( _mock_objective, config=config, search_alg=searcher, num_samples=1) trial = analysis.trials[0] diff --git a/python/ray/tune/trial_runner.py b/python/ray/tune/trial_runner.py index cf7c7d9edadb..e63f10a1b6dd 100644 --- a/python/ray/tune/trial_runner.py +++ b/python/ray/tune/trial_runner.py @@ -743,7 +743,7 @@ def _process_trial_result(self, trial, result): flat_result = flatten_dict(result) self._validate_result_metrics(flat_result) - + if self._stopper(trial.trial_id, result) or trial.should_stop(flat_result): result.update(done=True)