[tune/wandb] Use resume=False per default (#21892)

The WandbLoggingCallback is run on the driver side, with the experiment directory was the cwd. Using resume=True will pick up state from other trials (as the file name is global), and thus lead to warning messages. Thus, we should default to resume=False when using the callback. This PR also incorporates changes from #20966. Co-authored by: Queimo <[email protected]> Co-authored by: Karim <[email protected]>
ray-project · Jan 27, 2022 · 8dcd4a9 · 8dcd4a9
1 parent 634f897
commit 8dcd4a9
Showing 1 changed file with 9 additions and 3 deletions.
diff --git a/python/ray/tune/integration/wandb.py b/python/ray/tune/integration/wandb.py
@@ -195,7 +195,8 @@ def __init__(self, queue: Queue, exclude: List[str], to_config: List[str],
         self.kwargs = kwargs
 
     def run(self):
-        os.environ["WANDB_START_METHOD"] = "fork"
+        # Since we're running in a separate process already, use threads.
+        os.environ["WANDB_START_METHOD"] = "thread"
         wandb.init(*self.args, **self.kwargs)
         while True:
             result = self.queue.get()
@@ -350,7 +351,7 @@ def log_trial_start(self, trial: "Trial"):
         wandb_init_kwargs = dict(
             id=trial_id,
             name=trial_name,
-            resume=True,
+            resume=False,
             reinit=True,
             allow_val_change=True,
             group=wandb_group,
@@ -566,7 +567,12 @@ def __init__(self, config: Dict, *args, **kwargs):
             config=_config)
         wandb_init_kwargs.update(wandb_config)
 
-        os.environ["WANDB_START_METHOD"] = "fork"
+        # On windows, we can't fork
+        if os.name == "nt":
+            os.environ["WANDB_START_METHOD"] = "thread"
+        else:
+            os.environ["WANDB_START_METHOD"] = "fork"
+
         self.wandb = self._wandb.init(**wandb_init_kwargs)
 
     def stop(self):