Skip to content

Commit

Permalink
Use hydra.run.dir (not os.getcwd) for DDP subprocesses' run dir
Browse files Browse the repository at this point in the history
  • Loading branch information
nisheethlahoti committed Jul 24, 2023
1 parent 3d573d5 commit a4457f1
Showing 1 changed file with 3 additions and 2 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,7 @@ def _basic_subprocess_cmd() -> Sequence[str]:

def _hydra_subprocess_cmd(local_rank: int) -> Tuple[Sequence[str], str]:
import __main__ # local import to avoid https://github.com/Lightning-AI/lightning/issues/15218
from hydra.core.hydra_config import HydraConfig
from hydra.utils import get_original_cwd, to_absolute_path

# when user is using hydra find the absolute path
Expand All @@ -154,6 +155,6 @@ def _hydra_subprocess_cmd(local_rank: int) -> Tuple[Sequence[str], str]:
command += sys.argv[1:]

cwd = get_original_cwd()
os_cwd = f'"{os.getcwd()}"'
command += [f"hydra.run.dir={os_cwd}", f"hydra.job.name=train_ddp_process_{local_rank}"]
run_dir = f'"{HydraConfig.get().run.dir}"'
command += [f"hydra.run.dir={run_dir}", f"hydra.job.name=train_ddp_process_{local_rank}"]
return command, cwd

0 comments on commit a4457f1

Please sign in to comment.