[RLlib] Issue 31323: BC/MARWIL/CQL do work with multi-GPU (but config…

… validation prevents them from running in this mode). (ray-project#31393)

Signed-off-by: tmynn <[email protected]>

rllib/algorithms/cql/cql.py

@@ -134,9 +134,6 @@ def validate(self) -> None:
         # Call super's validation method.
         super().validate()
 
-        if self.num_gpus > 1:
-            raise ValueError("`num_gpus` > 1 not yet supported for CQL!")
-
         # CQL-torch performs the optimizer steps inside the loss function.
         # Using the multi-GPU optimizer will therefore not work (see multi-GPU
         # check above) and we must use the simple optimizer for now.

rllib/algorithms/cql/tests/test_cql.py

@@ -67,7 +67,7 @@ def test_cql_compilation(self):
                 evaluation_num_workers=2,
             )
             .rollouts(num_rollout_workers=0)
-            .reporting(min_time_s_per_iteration=0.0)
+            .reporting(min_time_s_per_iteration=0)
         )
         num_iterations = 4
 

rllib/algorithms/marwil/marwil.py

@@ -195,9 +195,6 @@ def validate(self) -> None:
         if self.beta < 0.0 or self.beta > 1.0:
             raise ValueError("`beta` must be within 0.0 and 1.0!")
 
-        if self.num_gpus > 1:
-            raise ValueError("`num_gpus` > 1 not yet supported for MARWIL!")
-
         if self.postprocess_inputs is False and self.beta > 0.0:
             raise ValueError(
                 "`postprocess_inputs` must be True for MARWIL (to "