feat: increase GLOO timeout [DET-3309] (#729)

When running on a large number of nodes (40+), the default
timeout of 30 seconds is not enough for all nodes to connect
with the chief process.

harness/determined/constants.py

@@ -59,3 +59,8 @@
 # Path for file that stores output of horovod auto-tuning. Only created when
 # horovod auto-tuning is enabled.
 HOROVOD_AUTOTUNE_LOG_FILEPATH = "/tmp/autotune_log.csv"
+
+# How many seconds GLOO waits for all tasks to connect before failing.
+# Increasing this from a default of 30 is necessary when there is a
+# large number of machines.
+HOROVOD_GLOO_TIMEOUT_SECONDS = 240

harness/determined/horovod.py

@@ -143,6 +143,8 @@ def create_run_command(
         create_hostlist_arg(num_gpus_per_machine, ip_addresses),
         "--start-timeout",
         str(constants.HOROVOD_STARTUP_TIMEOUT_SECONDS),
+        "--gloo-timeout-seconds",
+        str(constants.HOROVOD_GLOO_TIMEOUT_SECONDS),
     ]
     horovod_process_cmd.extend(create_network_interface_arg_if_specified(env, num_machines))
     horovod_process_cmd.extend(create_performance_args(env))

harness/tests/test_horovod.py

@@ -64,6 +64,8 @@ def test_create_run_command(
         "localhost:8,128.140.2.4:8",
         "--start-timeout",
         str(constants.HOROVOD_STARTUP_TIMEOUT_SECONDS),
+        "--gloo-timeout-seconds",
+        str(constants.HOROVOD_GLOO_TIMEOUT_SECONDS),
     ]
     if auto_tune:
         expected_horovod_run_cmd.extend(