From f44ad19d89e0e9fef1dccf353bb1222b454c5508 Mon Sep 17 00:00:00 2001 From: Balaji Veeramani Date: Wed, 11 Jan 2023 18:37:12 -0800 Subject: [PATCH 1/3] Initial commit Signed-off-by: Balaji Veeramani --- .buildkite/pipeline.build.yml | 3 +- python/ray/air/integrations/wandb.py | 55 ++++++++++++++++++---------- 2 files changed, 37 insertions(+), 21 deletions(-) diff --git a/.buildkite/pipeline.build.yml b/.buildkite/pipeline.build.yml index 84254c424a52..7adb440b9065 100644 --- a/.buildkite/pipeline.build.yml +++ b/.buildkite/pipeline.build.yml @@ -573,6 +573,7 @@ - pip uninstall -y ray && rm -rf /ray/python/ray/thirdparty_files - pushd /ray && git clean -f -f -x -d -e .whl -e python/ray/dashboard/client && popd - bazel clean --expunge + - python ./ci/env/setup_credentials.py wandb - ./ci/ci.sh build - label: ":octopus: Tune multinode tests" @@ -623,7 +624,7 @@ - pip install -U typing-extensions - HOROVOD_WITH_GLOO=1 HOROVOD_WITHOUT_MPI=1 HOROVOD_WITHOUT_MXNET=1 HOROVOD_WITH_TENSORFLOW=1 HOROVOD_WITH_PYTORCH=1 pip install horovod - ./ci/env/env_info.sh - - bazel test --config=ci $(./scripts/bazel_export_options) --test_tag_filters=compat_py36 + - bazel test --config=ci $(./scripts/bazel_export_options) --test_tag_filters=compat_py36 python/ray/tests/horovod/... python/ray/tests/lightgbm/... python/ray/tests/ml_py36_compat/... diff --git a/python/ray/air/integrations/wandb.py b/python/ray/air/integrations/wandb.py index c1240a94acc7..a6db1da5a158 100644 --- a/python/ray/air/integrations/wandb.py +++ b/python/ray/air/integrations/wandb.py @@ -440,6 +440,40 @@ class WandbLoggerCallback(LoggerCallback): ``LoggerCallback`` sends metrics to Wandb for automatic tracking and visualization. + Example: + + .. testcode:: + import random + + from ray import tune + from ray.air import session, RunConfig + from ray.air.integrations.wandb import WandbLoggerCallback + + + def train_func(config): + offset = random.random() / 5 + for epoch in range(2, config["epochs"]): + acc = 1 - (2 + config["lr"]) ** -epoch - random.random() / epoch - offset + loss = (2 + config["lr"]) ** -epoch + random.random() / epoch + offset + session.report({"acc": acc, "loss": loss}) + + + tuner = tune.Tuner( + train_func, + param_space={ + "lr": tune.grid_search([0.001, 0.01, 0.1, 1.0]), + "epochs": 10, + }, + run_config=RunConfig( + callbacks=[WandbLoggerCallback(project="Optimization_Project")] + ), + ) + results = tuner.fit() + + .. testoutput:: + :hide: + :options: +ELLIPSIS + Args: project: Name of the Wandb project. Mandatory. group: Name of the Wandb group. Defaults to the trainable @@ -464,26 +498,7 @@ class WandbLoggerCallback(LoggerCallback): Please see here for all other valid configuration settings: https://docs.wandb.ai/library/init - - Example: - - .. code-block:: python - - from ray.tune.logger import DEFAULT_LOGGERS - from ray.air.integrations.wandb import WandbLoggerCallback - tune.run( - train_fn, - config={ - # define search space here - "parameter_1": tune.choice([1, 2, 3]), - "parameter_2": tune.choice([4, 5, 6]), - }, - callbacks=[WandbLoggerCallback( - project="Optimization_Project", - api_key_file="/path/to/file", - log_config=True)]) - - """ + """ # noqa: E501 # Do not log these result keys _exclude_results = ["done", "should_checkpoint"] From 78d9c683e142b581fbc4f3739525ba02f66c45bb Mon Sep 17 00:00:00 2001 From: Balaji Veeramani Date: Wed, 18 Jan 2023 15:36:53 -0800 Subject: [PATCH 2/3] Fix test Signed-off-by: Balaji Veeramani --- .buildkite/pipeline.build.yml | 2 +- python/ray/air/integrations/wandb.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.buildkite/pipeline.build.yml b/.buildkite/pipeline.build.yml index 7adb440b9065..a2dba9bf628e 100644 --- a/.buildkite/pipeline.build.yml +++ b/.buildkite/pipeline.build.yml @@ -573,7 +573,7 @@ - pip uninstall -y ray && rm -rf /ray/python/ray/thirdparty_files - pushd /ray && git clean -f -f -x -d -e .whl -e python/ray/dashboard/client && popd - bazel clean --expunge - - python ./ci/env/setup_credentials.py wandb + - export WANDB_MODE=offline - ./ci/ci.sh build - label: ":octopus: Tune multinode tests" diff --git a/python/ray/air/integrations/wandb.py b/python/ray/air/integrations/wandb.py index a6db1da5a158..347616b75831 100644 --- a/python/ray/air/integrations/wandb.py +++ b/python/ray/air/integrations/wandb.py @@ -37,6 +37,7 @@ WANDB_ENV_VAR = "WANDB_API_KEY" WANDB_PROJECT_ENV_VAR = "WANDB_PROJECT_NAME" WANDB_GROUP_ENV_VAR = "WANDB_GROUP_NAME" +WANDB_MODE_ENV_VAR = "WANDB_MODE" # Hook that is invoked before wandb.init in the setup method of WandbLoggerCallback # to populate the API key if it isn't already set when initializing the callback. # It doesn't take in any arguments and returns the W&B API key. @@ -290,6 +291,9 @@ def _get_wandb_project(project: Optional[str] = None) -> Optional[str]: def _set_api_key(api_key_file: Optional[str] = None, api_key: Optional[str] = None): """Set WandB API key from `wandb_config`. Will pop the `api_key_file` and `api_key` keys from `wandb_config` parameter""" + if os.environ[WANDB_MODE_ENV_VAR] in {"offline", "disabled"}: + return + if api_key_file: if api_key: raise ValueError("Both WandB `api_key_file` and `api_key` set.") @@ -443,6 +447,7 @@ class WandbLoggerCallback(LoggerCallback): Example: .. testcode:: + import random from ray import tune @@ -474,6 +479,8 @@ def train_func(config): :hide: :options: +ELLIPSIS + ... + Args: project: Name of the Wandb project. Mandatory. group: Name of the Wandb group. Defaults to the trainable From e1abb9d23ab5826ed52d49dcc2a3b5e12f733bd7 Mon Sep 17 00:00:00 2001 From: Balaji Veeramani Date: Wed, 18 Jan 2023 18:40:10 -0800 Subject: [PATCH 3/3] Update wandb.py Signed-off-by: Balaji Veeramani --- python/ray/air/integrations/wandb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ray/air/integrations/wandb.py b/python/ray/air/integrations/wandb.py index d1e525388d2e..54e9571463e2 100644 --- a/python/ray/air/integrations/wandb.py +++ b/python/ray/air/integrations/wandb.py @@ -298,7 +298,7 @@ def _set_api_key(api_key_file: Optional[str] = None, api_key: Optional[str] = No 3) User already logged in to W&B (wandb.api.api_key set) 4) From external hook WANDB_SETUP_API_KEY_HOOK """ - if os.environ[WANDB_MODE_ENV_VAR] in {"offline", "disabled"}: + if os.environ.get(WANDB_MODE_ENV_VAR) in {"offline", "disabled"}: return if api_key_file: