From 945b0ace6265fd76619aa3b8a67ea668c354ee38 Mon Sep 17 00:00:00 2001 From: Antoni Baum Date: Tue, 22 Feb 2022 15:10:03 +0000 Subject: [PATCH 1/2] [tune] Note TPESampler performance issues in docs --- python/ray/tune/suggest/optuna.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/python/ray/tune/suggest/optuna.py b/python/ray/tune/suggest/optuna.py index 117d67e49297..16439c8aaf88 100644 --- a/python/ray/tune/suggest/optuna.py +++ b/python/ray/tune/suggest/optuna.py @@ -121,6 +121,16 @@ class OptunaSearch(Searcher): draw hyperparameter configurations. Defaults to ``MOTPESampler`` for multi-objective optimization with Optuna<2.9.0, and ``TPESampler`` in every other case. + + ..warning:: + Please note that with Optuna 2.10.0 and earlier + default ``MOTPESampler``/``TPESampler`` suffer + from performance issues when dealing with a large number of + completed trials (approx. >100). This will manifest as + a delay when suggesting new configurations. + This is an Optuna issue and may be fixed in a future + Optuna release. + seed (int): Seed to initialize sampler with. This parameter is only used when ``sampler=None``. In all other cases, the sampler you pass should be initialized with the seed already. From ada558089ceb77d07eb6c09ecab0a17043c513d3 Mon Sep 17 00:00:00 2001 From: Antoni Baum Date: Tue, 22 Feb 2022 18:54:54 +0000 Subject: [PATCH 2/2] Fix --- python/ray/tune/suggest/optuna.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/ray/tune/suggest/optuna.py b/python/ray/tune/suggest/optuna.py index 16439c8aaf88..faa4e5e927e8 100644 --- a/python/ray/tune/suggest/optuna.py +++ b/python/ray/tune/suggest/optuna.py @@ -122,7 +122,7 @@ class OptunaSearch(Searcher): for multi-objective optimization with Optuna<2.9.0, and ``TPESampler`` in every other case. - ..warning:: + .. warning:: Please note that with Optuna 2.10.0 and earlier default ``MOTPESampler``/``TPESampler`` suffer from performance issues when dealing with a large number of @@ -141,7 +141,7 @@ class OptunaSearch(Searcher): needing to re-compute the trial. Must be the same length as points_to_evaluate. - ..warning:: + .. warning:: When using ``evaluated_rewards``, the search space ``space`` must be provided as a :class:`dict` with parameter names as keys and ``optuna.distributions`` instances as values. The