From 7784e43ec93559fee0f1bd7c4cb34f594ef34c22 Mon Sep 17 00:00:00 2001 From: Archit Kulkarni Date: Wed, 14 Sep 2022 16:35:59 -0700 Subject: [PATCH 1/5] Add docs for custom resources with Serve --- doc/source/serve/scaling-and-resource-allocation.md | 11 +++++++++++ python/ray/serve/api.py | 4 +++- python/ray/serve/config.py | 3 ++- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/doc/source/serve/scaling-and-resource-allocation.md b/doc/source/serve/scaling-and-resource-allocation.md index a7d6b79d444f..4feea18daec1 100644 --- a/doc/source/serve/scaling-and-resource-allocation.md +++ b/doc/source/serve/scaling-and-resource-allocation.md @@ -123,6 +123,17 @@ def func_2(*args): In this example, each replica of each deployment will be allocated 0.5 GPUs. The same can be done to multiplex over CPUs, using `"num_cpus"`. +### Custom Resources + +You can also specify [custom resources](custom-resources) in `ray_actor_options`, for example to ensure that a deployment is scheduled on a specific node. +For example, if you have a deployment that requires 2 units of the `"custom_resource"` resource, you can specify it like this: + +```python +@serve.deployment(ray_actor_options={"resources": {"custom_resource": 2}}) +def func(*args): + return do_something_with_my_custom_resource() +``` + (serve-omp-num-threads)= ## Configuring Parallelism with OMP_NUM_THREADS diff --git a/python/ray/serve/api.py b/python/ray/serve/api.py index 611b665a408c..7c527bc48b99 100644 --- a/python/ray/serve/api.py +++ b/python/ray/serve/api.py @@ -317,7 +317,9 @@ def deployment( a '/' unless they're the root (just '/'), which acts as a catch-all. ray_actor_options: Options to be passed to the Ray actor - constructor such as resource requirements. + constructor such as resource requirements. Valid options are + `accellerator_type`, `memory`, `num_cpus`, `num_gpus`, + `object_store_memory`, `resources`, and `runtime_env`. user_config (Optional[Any]): Config to pass to the reconfigure method of the deployment. This can be updated dynamically without changing the version of the deployment and diff --git a/python/ray/serve/config.py b/python/ray/serve/config.py index c11d233d17d7..34187b99e338 100644 --- a/python/ray/serve/config.py +++ b/python/ray/serve/config.py @@ -390,7 +390,8 @@ def _validate_ray_actor_options(self) -> None: f'Got invalid type "{type(self.ray_actor_options)}" for ' "ray_actor_options. Expected a dictionary." ) - + # Please keep this in sync with the docstring for the ray_actor_options + # kwarg in api.py. allowed_ray_actor_options = { # Resource options "accelerator_type", From 085767bead021feb31971fb2d2be1317a6054403 Mon Sep 17 00:00:00 2001 From: Archit Kulkarni Date: Thu, 15 Sep 2022 10:00:36 -0700 Subject: [PATCH 2/5] Update python/ray/serve/api.py Co-authored-by: shrekris-anyscale <92341594+shrekris-anyscale@users.noreply.github.com> Signed-off-by: Archit Kulkarni --- python/ray/serve/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ray/serve/api.py b/python/ray/serve/api.py index 7c527bc48b99..7a2754954c93 100644 --- a/python/ray/serve/api.py +++ b/python/ray/serve/api.py @@ -318,7 +318,7 @@ def deployment( catch-all. ray_actor_options: Options to be passed to the Ray actor constructor such as resource requirements. Valid options are - `accellerator_type`, `memory`, `num_cpus`, `num_gpus`, + `accelerator_type`, `memory`, `num_cpus`, `num_gpus`, `object_store_memory`, `resources`, and `runtime_env`. user_config (Optional[Any]): Config to pass to the reconfigure method of the deployment. This can be updated From 0cf860eb5ddc055c6b6f73a68932590c52022209 Mon Sep 17 00:00:00 2001 From: Archit Kulkarni Date: Fri, 16 Sep 2022 15:08:29 -0700 Subject: [PATCH 3/5] Add custom resources tag --- doc/source/ray-core/configure.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/source/ray-core/configure.rst b/doc/source/ray-core/configure.rst index d204ddb6fb78..9dd4c4292783 100644 --- a/doc/source/ray-core/configure.rst +++ b/doc/source/ray-core/configure.rst @@ -12,6 +12,8 @@ and from the command line. Take a look at the ``ray.init`` `documentation .. important:: For the multi-node setting, you must first run ``ray start`` on the command line to start the Ray cluster services on the machine before ``ray.init`` in Python to connect to the cluster services. On a single machine, you can run ``ray.init()`` without ``ray start``, which will both start the Ray cluster services and connect to them. +(cluster-resources)= + Cluster Resources ----------------- From 112fec2b9d882ec49caf06e7344b78cd8af35b06 Mon Sep 17 00:00:00 2001 From: Archit Kulkarni Date: Fri, 16 Sep 2022 15:18:16 -0700 Subject: [PATCH 4/5] Add accelerator types and full list --- doc/source/ray-core/miscellaneous.rst | 2 ++ .../serve/scaling-and-resource-allocation.md | 14 +++++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/doc/source/ray-core/miscellaneous.rst b/doc/source/ray-core/miscellaneous.rst index c75777ea86e9..0d6d61686544 100644 --- a/doc/source/ray-core/miscellaneous.rst +++ b/doc/source/ray-core/miscellaneous.rst @@ -75,6 +75,8 @@ appear as the task name in the logs. .. image:: images/task_name_dashboard.png +(accelerator-types)= + Accelerator Types ------------------ diff --git a/doc/source/serve/scaling-and-resource-allocation.md b/doc/source/serve/scaling-and-resource-allocation.md index 4feea18daec1..cc0c623e5661 100644 --- a/doc/source/serve/scaling-and-resource-allocation.md +++ b/doc/source/serve/scaling-and-resource-allocation.md @@ -123,7 +123,7 @@ def func_2(*args): In this example, each replica of each deployment will be allocated 0.5 GPUs. The same can be done to multiplex over CPUs, using `"num_cpus"`. -### Custom Resources +### Custom Resources, Accelerator types, and more You can also specify [custom resources](custom-resources) in `ray_actor_options`, for example to ensure that a deployment is scheduled on a specific node. For example, if you have a deployment that requires 2 units of the `"custom_resource"` resource, you can specify it like this: @@ -134,6 +134,18 @@ def func(*args): return do_something_with_my_custom_resource() ``` +You can also specify [accelerator types](accelerator-types) via the `accelerator_type` parameter in `ray_actor_options`. + +Below is the full list of supported options in `ray_actor_options`; please see the relevant Ray Core documentation for more details about each option: + +- `accelerator_type` +- `memory` +- `num_cpus` +- `num_gpus` +- `object_store_memory` +- `resources` +- `runtime_env` + (serve-omp-num-threads)= ## Configuring Parallelism with OMP_NUM_THREADS From 18039b3ce8598b147bff8dc39d8cd16753c6160d Mon Sep 17 00:00:00 2001 From: Archit Kulkarni Date: Mon, 19 Sep 2022 09:56:32 -0700 Subject: [PATCH 5/5] Fix sphinx references --- doc/source/ray-core/configure.rst | 2 +- doc/source/ray-core/miscellaneous.rst | 2 +- doc/source/serve/scaling-and-resource-allocation.md | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/source/ray-core/configure.rst b/doc/source/ray-core/configure.rst index 9dd4c4292783..46c8d38f9421 100644 --- a/doc/source/ray-core/configure.rst +++ b/doc/source/ray-core/configure.rst @@ -12,7 +12,7 @@ and from the command line. Take a look at the ``ray.init`` `documentation .. important:: For the multi-node setting, you must first run ``ray start`` on the command line to start the Ray cluster services on the machine before ``ray.init`` in Python to connect to the cluster services. On a single machine, you can run ``ray.init()`` without ``ray start``, which will both start the Ray cluster services and connect to them. -(cluster-resources)= +.. _cluster-resources: Cluster Resources ----------------- diff --git a/doc/source/ray-core/miscellaneous.rst b/doc/source/ray-core/miscellaneous.rst index 0d6d61686544..9c20ce31282c 100644 --- a/doc/source/ray-core/miscellaneous.rst +++ b/doc/source/ray-core/miscellaneous.rst @@ -75,7 +75,7 @@ appear as the task name in the logs. .. image:: images/task_name_dashboard.png -(accelerator-types)= +.. _accelerator-types: Accelerator Types ------------------ diff --git a/doc/source/serve/scaling-and-resource-allocation.md b/doc/source/serve/scaling-and-resource-allocation.md index cc0c623e5661..47148b8213aa 100644 --- a/doc/source/serve/scaling-and-resource-allocation.md +++ b/doc/source/serve/scaling-and-resource-allocation.md @@ -125,7 +125,7 @@ In this example, each replica of each deployment will be allocated 0.5 GPUs. Th ### Custom Resources, Accelerator types, and more -You can also specify [custom resources](custom-resources) in `ray_actor_options`, for example to ensure that a deployment is scheduled on a specific node. +You can also specify {ref}`custom resources ` in `ray_actor_options`, for example to ensure that a deployment is scheduled on a specific node. For example, if you have a deployment that requires 2 units of the `"custom_resource"` resource, you can specify it like this: ```python @@ -134,7 +134,7 @@ def func(*args): return do_something_with_my_custom_resource() ``` -You can also specify [accelerator types](accelerator-types) via the `accelerator_type` parameter in `ray_actor_options`. +You can also specify {ref}`accelerator types ` via the `accelerator_type` parameter in `ray_actor_options`. Below is the full list of supported options in `ray_actor_options`; please see the relevant Ray Core documentation for more details about each option: