add gather_use_object arguments

src/transformers/trainer.py

@@ -3831,6 +3831,13 @@ def evaluation_loop(
             if is_torch_xla_available():
                 xm.mark_step()
 
+            if "use_gather_object" in inspect.signature(
+                self.gather_function
+            ).parameters.keys() and is_accelerate_available("0.30.0"):
+                self.gather_function = functools.partial(
+                    self.gather_function, use_gather_object=self.args.eval_use_gather_object
+                )
+
             # Update containers
             if losses is not None:
                 losses = self.gather_function((losses.repeat(batch_size)))

src/transformers/training_args.py

@@ -775,6 +775,9 @@ class TrainingArguments:
 
         eval_on_start(`bool`, *optional*, defaults to `False`):
             Whether to perform a evaluation step (sanity check) before the training to ensure the validation steps works correctly.
+
+        eval_use_gather_object(`bool`, *optional*, defaults to `False`):
+            Whether to run recursively gather object in a nested list/tuple/dictionary of objects from all devices.
     """
 
     framework = "pt"
@@ -1465,6 +1468,13 @@ class TrainingArguments:
         },
     )
 
+    eval_use_gather_object: Optional[bool] = field(
+        default=False,
+        metadata={
+            "help": "Whether to run recursively gather object in a nested list/tuple/dictionary of objects from all devices."
+        },
+    )
+
     def __post_init__(self):
         # Parse in args that could be `dict` sent in from the CLI as a string
         for field in _VALID_DICT_FIELDS: