[tune] Optuna should ignore additional results after trial termination

python/ray/tune/suggest/optuna.py

@@ -327,6 +327,8 @@ def __init__(
         self._sampler = sampler
         self._seed = seed
 
+        self._completed_trials = set()
+
         self._ot_trials = {}
         self._ot_study = None
         if self._space:
@@ -474,6 +476,12 @@ def on_trial_result(self, trial_id: str, result: Dict):
             # Optuna doesn't support incremental results
             # for multi-objective optimization
             return
+        if trial_id in self._completed_trials:
+            logger.warning(
+                f"Received additional result for trial {trial_id}, but "
+                f"it already finished. Result: {result}"
+            )
+            return
         metric = result[self.metric]
         step = result[TRAINING_ITERATION]
         ot_trial = self._ot_trials[trial_id]
@@ -482,6 +490,13 @@ def on_trial_result(self, trial_id: str, result: Dict):
     def on_trial_complete(
         self, trial_id: str, result: Optional[Dict] = None, error: bool = False
     ):
+        if trial_id in self._completed_trials:
+            logger.warning(
+                f"Received additional completion for trial {trial_id}, but "
+                f"it already finished. Result: {result}"
+            )
+            return
+
         ot_trial = self._ot_trials[trial_id]
 
         if result:
@@ -499,9 +514,11 @@ def on_trial_complete(
                 ot_trial_state = OptunaTrialState.PRUNED
         try:
             self._ot_study.tell(ot_trial, val, state=ot_trial_state)
-        except ValueError as exc:
+        except Exception as exc:
             logger.warning(exc)  # E.g. if NaN was reported
 
+        self._completed_trials.add(trial_id)
+
     def add_evaluated_point(
         self,
         parameters: Dict,

python/ray/tune/tests/test_searchers.py

@@ -249,6 +249,25 @@ def testOptuna(self):
         best_trial = out.best_trial
         self.assertLessEqual(best_trial.config["report"], 2.0)
 
+    def testOptunaReportTooOften(self):
+        from ray.tune.suggest.optuna import OptunaSearch
+        from optuna.samplers import RandomSampler
+
+        searcher = OptunaSearch(
+            sampler=RandomSampler(seed=1234),
+            space=OptunaSearch.convert_search_space(self.config),
+            metric="metric",
+            mode="max",
+        )
+        searcher.suggest("trial_1")
+        searcher.on_trial_result("trial_1", {"training_iteration": 1, "metric": 1})
+        searcher.on_trial_complete("trial_1", {"training_iteration": 2, "metric": 1})
+
+        # Report after complete should not fail
+        searcher.on_trial_result("trial_1", {"training_iteration": 3, "metric": 1})
+
+        searcher.on_trial_complete("trial_1", {"training_iteration": 4, "metric": 1})
+
     def testSkopt(self):
         from ray.tune.suggest.skopt import SkOptSearch