[Data] [Release Test] Add AWS ACCESS_DENIED as retryable exception fo…

…r multi-node Data+Train benchmarks cherrypick #47232 Signed-off-by: Lonnie Liu <[email protected]>
ray-project · Aug 21, 2024 · 563e206 · 563e206
1 parent 4659bab
commit 563e206
Showing 1 changed file with 6 additions and 1 deletion.
diff --git a/release/nightly_tests/dataset/multi_node_train_benchmark.py b/release/nightly_tests/dataset/multi_node_train_benchmark.py
@@ -571,8 +571,13 @@ def __iter__(self):
 def benchmark_code(
     args,
 ):
+    ctx = ray.data.DataContext.get_current()
+    # This release test runs into ACCESS_DENIED errors fairly often.
+    # We add ACCESS_DENIED as a retryable exception type to avoid flakiness.
+    # See for more details: https://github.com/ray-project/ray/issues/47230
+    ctx.retried_io_errors.append("AWS Error ACCESS_DENIED")
+
     if args.target_max_block_size_mb is not None:
-        ctx = ray.data.DataContext.get_current()
         ctx.target_max_block_size = args.target_max_block_size_mb * 1024 * 1024
 
     cache_input_ds = args.cache_input_ds