Reconfigure limit_val_batches only for int (NVIDIA#8099)

Signed-off-by: Abhishree <[email protected]> Signed-off-by: Piotr Żelasko <[email protected]>
pzelasko · Jan 3, 2024 · d931432 · d931432
1 parent 516b4be
commit d931432
Showing 1 changed file with 3 additions and 2 deletions.
diff --git a/nemo/collections/nlp/models/language_modeling/megatron_base_model.py b/nemo/collections/nlp/models/language_modeling/megatron_base_model.py
@@ -246,8 +246,9 @@ def _reconfigure_val_batches(self):
         """
         Reconfigure trainer.limit_val_batches for pretraining
         """
-        # Override limit_val_batches to be a multiple of num microbatches and so there are limit_val_batches//num_micro_batches num of global batches
-        self.trainer.limit_val_batches *= get_num_microbatches()
+        if isinstance(self.trainer.limit_val_batches, int):
+            # Override limit_val_batches to be a multiple of num microbatches and so there are limit_val_batches//num_micro_batches num of global batches
+            self.trainer.limit_val_batches *= get_num_microbatches()
         # Override num sanity steps to be a multiple of num of microbatches
         self.trainer.num_sanity_val_steps *= get_num_microbatches()