diff --git a/python/ray/tune/tests/test_progress_reporter.py b/python/ray/tune/tests/test_progress_reporter.py index d785d0799ec3..c75750bad0fc 100644 --- a/python/ray/tune/tests/test_progress_reporter.py +++ b/python/ray/tune/tests/test_progress_reporter.py @@ -1,5 +1,6 @@ import collections import os +import regex as re import unittest from unittest.mock import MagicMock, Mock, patch @@ -242,14 +243,19 @@ def f(config): "with parameters={'do': 'complete'}. This trial completed.\n" ) -VERBOSE_TRIAL_NORM_2 = """ -Trial train_xxxxx_00001 reported _metric=6 with parameters={'do': 'once'}. -Trial train_xxxxx_00001 completed. Last result: _metric=6 -""" +# NOTE: We use Regex for `VERBOSE_TRIAL_NORM_2` to make the test deterministic. +# `"Trial train_xxxxx_00001 reported..."` and `"Trial train_xxxxx_00001 completed..."` +# are printed in separate calls. Sometimes, a status update is printed between the +# calls. For more information, see #29693. +VERBOSE_TRIAL_NORM_2_PATTERN = ( + r"Trial train_xxxxx_00001 reported _metric=6 with parameters=\{'do': 'once'\}\.\n" + r"(?s).*" + r"Trial train_xxxxx_00001 completed\. Last result: _metric=6\n" +) -VERBOSE_TRIAL_NORM_3 = """ -Trial train_xxxxx_00002 reported acc=7 with parameters={'do': 'twice'}. -""" +VERBOSE_TRIAL_NORM_3 = ( + "Trial train_xxxxx_00002 reported acc=7 with parameters={'do': 'twice'}.\n" +) VERBOSE_TRIAL_NORM_4 = ( "Trial train_xxxxx_00002 reported acc=8 " @@ -639,7 +645,7 @@ def testVerboseReporting(self): self.assertNotIn(VERBOSE_EXP_OUT_1, output) self.assertNotIn(VERBOSE_EXP_OUT_2, output) self.assertNotIn(VERBOSE_TRIAL_NORM_1, output) - self.assertNotIn(VERBOSE_TRIAL_NORM_2, output) + self.assertIsNone(re.search(VERBOSE_TRIAL_NORM_2_PATTERN, output)) self.assertNotIn(VERBOSE_TRIAL_NORM_3, output) self.assertNotIn(VERBOSE_TRIAL_NORM_4, output) self.assertNotIn(VERBOSE_TRIAL_DETAIL, output) @@ -655,7 +661,7 @@ def testVerboseReporting(self): self.assertIn(VERBOSE_EXP_OUT_1, output) self.assertIn(VERBOSE_EXP_OUT_2, output) self.assertNotIn(VERBOSE_TRIAL_NORM_1, output) - self.assertNotIn(VERBOSE_TRIAL_NORM_2, output) + self.assertIsNone(re.search(VERBOSE_TRIAL_NORM_2_PATTERN, output)) self.assertNotIn(VERBOSE_TRIAL_NORM_3, output) self.assertNotIn(VERBOSE_TRIAL_NORM_4, output) self.assertNotIn(VERBOSE_TRIAL_DETAIL, output) @@ -671,7 +677,7 @@ def testVerboseReporting(self): self.assertIn(VERBOSE_EXP_OUT_1, output) self.assertIn(VERBOSE_EXP_OUT_2, output) self.assertIn(VERBOSE_TRIAL_NORM_1, output) - self.assertIn(VERBOSE_TRIAL_NORM_2, output) + self.assertIsNotNone(re.search(VERBOSE_TRIAL_NORM_2_PATTERN, output)) self.assertIn(VERBOSE_TRIAL_NORM_3, output) self.assertIn(VERBOSE_TRIAL_NORM_4, output) self.assertNotIn(VERBOSE_TRIAL_DETAIL, output) @@ -687,7 +693,7 @@ def testVerboseReporting(self): self.assertIn(VERBOSE_EXP_OUT_1, output) self.assertIn(VERBOSE_EXP_OUT_2, output) self.assertNotIn(VERBOSE_TRIAL_NORM_1, output) - self.assertNotIn(VERBOSE_TRIAL_NORM_2, output) + self.assertIsNone(re.search(VERBOSE_TRIAL_NORM_2_PATTERN, output)) self.assertNotIn(VERBOSE_TRIAL_NORM_3, output) self.assertNotIn(VERBOSE_TRIAL_NORM_4, output) self.assertIn(VERBOSE_TRIAL_DETAIL, output)