From 5ef5251c71c06dad7df909dd5e556bd58b620089 Mon Sep 17 00:00:00 2001 From: Dmitri Gekhtman Date: Mon, 19 Dec 2022 23:23:07 +0000 Subject: [PATCH 01/23] Schedule CPU tasks when there are no GPU worker types. Signed-off-by: Dmitri Gekhtman --- .../_private/resource_demand_scheduler.py | 19 +++++++-- .../tests/test_autoscaler_fake_multinode.py | 39 +++++++++++++++++++ 2 files changed, 55 insertions(+), 3 deletions(-) diff --git a/python/ray/autoscaler/_private/resource_demand_scheduler.py b/python/ray/autoscaler/_private/resource_demand_scheduler.py index 0154d7d3ff66..d37a33a30ec7 100644 --- a/python/ray/autoscaler/_private/resource_demand_scheduler.py +++ b/python/ray/autoscaler/_private/resource_demand_scheduler.py @@ -209,8 +209,15 @@ def get_nodes_to_launch( Dict of count to add for each node type, and residual of resources that still cannot be fulfilled. """ + # Does every node type have gpus? + all_gpu_node_types = all( + node_type_config.get("resources").get("GPU", 0) != 0 + for node_type_config in self.node_types.values() + ) utilization_scorer = partial( - self.utilization_scorer, node_availability_summary=node_availability_summary + self.utilization_scorer, + node_availability_summary=node_availability_summary, + all_gpu_node_types=all_gpu_node_types, ) self._update_node_resources_from_runtime(nodes, max_resources_by_ip) @@ -806,6 +813,7 @@ def _resource_based_utilization_scorer( resources: List[ResourceDict], *, node_availability_summary: NodeAvailabilitySummary, + all_gpu_node_types: bool = False ) -> Optional[Tuple[float, float]]: remaining = copy.deepcopy(node_resources) is_gpu_node = "GPU" in node_resources and node_resources["GPU"] > 0 @@ -814,7 +822,10 @@ def _resource_based_utilization_scorer( # Avoid launching GPU nodes if there aren't any GPU tasks at all. Note that # if there *is* a GPU task, then CPU tasks can be scheduled as well. if AUTOSCALER_CONSERVE_GPU_NODES: - if is_gpu_node and not any_gpu_task: + # If all available node types have GPUs, don't avoid upscaling. + if all_gpu_node_types: + pass + elif is_gpu_node and not any_gpu_task: return None fittable = [] @@ -862,9 +873,11 @@ def _default_utilization_scorer( node_type: str, *, node_availability_summary: NodeAvailabilitySummary, + all_gpu_node_types: bool = False ): return _resource_based_utilization_scorer( - node_resources, resources, node_availability_summary=node_availability_summary + node_resources, resources, node_availability_summary=node_availability_summary, + all_gpu_node_types=all_gpu_node_types ) diff --git a/python/ray/tests/test_autoscaler_fake_multinode.py b/python/ray/tests/test_autoscaler_fake_multinode.py index 2b01bfcc38b7..de065e43b515 100644 --- a/python/ray/tests/test_autoscaler_fake_multinode.py +++ b/python/ray/tests/test_autoscaler_fake_multinode.py @@ -1,3 +1,4 @@ +import copy import pytest import platform @@ -86,6 +87,44 @@ def ping(self): cluster.shutdown() +def test_autoscaler_all_gpu_node_types(): + """Validates that CPU tasks still trigger upscaling + when all available node types have GPUs. + """ + gpu_node_type_1 = { + "resources": { + "CPU": 1, + "GPU": 1, + }, + "node_config": {}, + "min_workers": 0, + "max_workers": 1, + }, + gpu_node_type_2 = copy.deepcopy(gpu_node_type_1) + + cluster = AutoscalingCluster( + head_resources={"CPU": 0, "GPU": 1}, + worker_node_types={ + "gpu_node_type_1": gpu_node_type_1, + "gpu_node_type_2": gpu_node_type_2, + } + ) + + try: + cluster.start() + ray.init("auto") + + @ray.remote(num_cpus=1) + def task(): + return True + + assert ray.get(task.remote(), timeout=60), "Failed to schedule CPU task." + ray.shutdown() + + finally: + cluster.shutdown() + + if __name__ == "__main__": import os import sys From 33874a851f8d00695d942fccfb927028e8bbd0bd Mon Sep 17 00:00:00 2001 From: Dmitri Gekhtman Date: Mon, 19 Dec 2022 23:29:15 +0000 Subject: [PATCH 02/23] Only check non-head types. Signed-off-by: Dmitri Gekhtman --- python/ray/autoscaler/_private/resource_demand_scheduler.py | 6 ++++-- python/ray/tests/test_autoscaler_fake_multinode.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/python/ray/autoscaler/_private/resource_demand_scheduler.py b/python/ray/autoscaler/_private/resource_demand_scheduler.py index d37a33a30ec7..a924ee46a653 100644 --- a/python/ray/autoscaler/_private/resource_demand_scheduler.py +++ b/python/ray/autoscaler/_private/resource_demand_scheduler.py @@ -209,10 +209,12 @@ def get_nodes_to_launch( Dict of count to add for each node type, and residual of resources that still cannot be fulfilled. """ - # Does every node type have gpus? + # Does every worker node type have gpus? + # (More precisely, does every non-head node type have gpus?) all_gpu_node_types = all( node_type_config.get("resources").get("GPU", 0) != 0 - for node_type_config in self.node_types.values() + for node_type, node_type_config in self.node_types.items() + if node_type != self.head_node_type ) utilization_scorer = partial( self.utilization_scorer, diff --git a/python/ray/tests/test_autoscaler_fake_multinode.py b/python/ray/tests/test_autoscaler_fake_multinode.py index de065e43b515..b2fd7618d515 100644 --- a/python/ray/tests/test_autoscaler_fake_multinode.py +++ b/python/ray/tests/test_autoscaler_fake_multinode.py @@ -103,7 +103,7 @@ def test_autoscaler_all_gpu_node_types(): gpu_node_type_2 = copy.deepcopy(gpu_node_type_1) cluster = AutoscalingCluster( - head_resources={"CPU": 0, "GPU": 1}, + head_resources={"CPU": 0}, worker_node_types={ "gpu_node_type_1": gpu_node_type_1, "gpu_node_type_2": gpu_node_type_2, From 89914f1f9bba6620ef22dfa81fc837b640200106 Mon Sep 17 00:00:00 2001 From: Dmitri Gekhtman Date: Mon, 19 Dec 2022 23:32:06 +0000 Subject: [PATCH 03/23] Lint. Signed-off-by: Dmitri Gekhtman --- .../_private/resource_demand_scheduler.py | 10 ++++++---- .../tests/test_autoscaler_fake_multinode.py | 20 ++++++++++--------- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/python/ray/autoscaler/_private/resource_demand_scheduler.py b/python/ray/autoscaler/_private/resource_demand_scheduler.py index a924ee46a653..809c926be4e4 100644 --- a/python/ray/autoscaler/_private/resource_demand_scheduler.py +++ b/python/ray/autoscaler/_private/resource_demand_scheduler.py @@ -815,7 +815,7 @@ def _resource_based_utilization_scorer( resources: List[ResourceDict], *, node_availability_summary: NodeAvailabilitySummary, - all_gpu_node_types: bool = False + all_gpu_node_types: bool = False, ) -> Optional[Tuple[float, float]]: remaining = copy.deepcopy(node_resources) is_gpu_node = "GPU" in node_resources and node_resources["GPU"] > 0 @@ -875,11 +875,13 @@ def _default_utilization_scorer( node_type: str, *, node_availability_summary: NodeAvailabilitySummary, - all_gpu_node_types: bool = False + all_gpu_node_types: bool = False, ): return _resource_based_utilization_scorer( - node_resources, resources, node_availability_summary=node_availability_summary, - all_gpu_node_types=all_gpu_node_types + node_resources, + resources, + node_availability_summary=node_availability_summary, + all_gpu_node_types=all_gpu_node_types, ) diff --git a/python/ray/tests/test_autoscaler_fake_multinode.py b/python/ray/tests/test_autoscaler_fake_multinode.py index b2fd7618d515..33a23c489b14 100644 --- a/python/ray/tests/test_autoscaler_fake_multinode.py +++ b/python/ray/tests/test_autoscaler_fake_multinode.py @@ -91,15 +91,17 @@ def test_autoscaler_all_gpu_node_types(): """Validates that CPU tasks still trigger upscaling when all available node types have GPUs. """ - gpu_node_type_1 = { - "resources": { - "CPU": 1, - "GPU": 1, + gpu_node_type_1 = ( + { + "resources": { + "CPU": 1, + "GPU": 1, + }, + "node_config": {}, + "min_workers": 0, + "max_workers": 1, }, - "node_config": {}, - "min_workers": 0, - "max_workers": 1, - }, + ) gpu_node_type_2 = copy.deepcopy(gpu_node_type_1) cluster = AutoscalingCluster( @@ -107,7 +109,7 @@ def test_autoscaler_all_gpu_node_types(): worker_node_types={ "gpu_node_type_1": gpu_node_type_1, "gpu_node_type_2": gpu_node_type_2, - } + }, ) try: From 425e714a832acdc9ea5de571c455e422816f0434 Mon Sep 17 00:00:00 2001 From: Dmitri Gekhtman Date: Mon, 19 Dec 2022 23:35:53 +0000 Subject: [PATCH 04/23] Comment Signed-off-by: Dmitri Gekhtman --- python/ray/tests/test_autoscaler_fake_multinode.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ray/tests/test_autoscaler_fake_multinode.py b/python/ray/tests/test_autoscaler_fake_multinode.py index 33a23c489b14..8b9532bdf132 100644 --- a/python/ray/tests/test_autoscaler_fake_multinode.py +++ b/python/ray/tests/test_autoscaler_fake_multinode.py @@ -89,7 +89,7 @@ def ping(self): def test_autoscaler_all_gpu_node_types(): """Validates that CPU tasks still trigger upscaling - when all available node types have GPUs. + when all available non-head node types have GPUs. """ gpu_node_type_1 = ( { From c519f802b011e7815e36b6ce69b23416ba4789ac Mon Sep 17 00:00:00 2001 From: Dmitri Gekhtman Date: Tue, 20 Dec 2022 00:07:32 +0000 Subject: [PATCH 05/23] Re-order. Signed-off-by: Dmitri Gekhtman --- python/ray/autoscaler/_private/resource_demand_scheduler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/ray/autoscaler/_private/resource_demand_scheduler.py b/python/ray/autoscaler/_private/resource_demand_scheduler.py index 809c926be4e4..b3d3440550dd 100644 --- a/python/ray/autoscaler/_private/resource_demand_scheduler.py +++ b/python/ray/autoscaler/_private/resource_demand_scheduler.py @@ -209,6 +209,8 @@ def get_nodes_to_launch( Dict of count to add for each node type, and residual of resources that still cannot be fulfilled. """ + # The following function mutates self.node_types. + self._update_node_resources_from_runtime(nodes, max_resources_by_ip) # Does every worker node type have gpus? # (More precisely, does every non-head node type have gpus?) all_gpu_node_types = all( @@ -221,7 +223,6 @@ def get_nodes_to_launch( node_availability_summary=node_availability_summary, all_gpu_node_types=all_gpu_node_types, ) - self._update_node_resources_from_runtime(nodes, max_resources_by_ip) node_resources: List[ResourceDict] node_type_counts: Dict[NodeType, int] From 4401a1a12be1a7fab8c00d96e00f40ee4181370d Mon Sep 17 00:00:00 2001 From: Dmitri Gekhtman Date: Tue, 20 Dec 2022 00:12:56 +0000 Subject: [PATCH 06/23] style Signed-off-by: Dmitri Gekhtman --- .../_private/resource_demand_scheduler.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/python/ray/autoscaler/_private/resource_demand_scheduler.py b/python/ray/autoscaler/_private/resource_demand_scheduler.py index b3d3440550dd..bb1539d63596 100644 --- a/python/ray/autoscaler/_private/resource_demand_scheduler.py +++ b/python/ray/autoscaler/_private/resource_demand_scheduler.py @@ -824,12 +824,15 @@ def _resource_based_utilization_scorer( # Avoid launching GPU nodes if there aren't any GPU tasks at all. Note that # if there *is* a GPU task, then CPU tasks can be scheduled as well. - if AUTOSCALER_CONSERVE_GPU_NODES: - # If all available node types have GPUs, don't avoid upscaling. - if all_gpu_node_types: - pass - elif is_gpu_node and not any_gpu_task: - return None + if not AUTOSCALER_CONSERVE_GPU_NODES: + # Skip GPU node avoidance if this is explicitly disabled. + pass + elif all_gpu_node_types: + # If ALL available non-head node types have GPUs, don't avoid upscaling. + pass + elif is_gpu_node and not any_gpu_task: + # Avoid upscaling the gpu node if the workload does not require GPUs. + return None fittable = [] resource_types = set() From a82141586bf2accc55c1106025aa55155f9feb39 Mon Sep 17 00:00:00 2001 From: Dmitri Gekhtman Date: Tue, 20 Dec 2022 00:16:30 +0000 Subject: [PATCH 07/23] Revert resource demand scheduler. Signed-off-by: Dmitri Gekhtman --- .../_private/resource_demand_scheduler.py | 33 ++++--------------- 1 file changed, 6 insertions(+), 27 deletions(-) diff --git a/python/ray/autoscaler/_private/resource_demand_scheduler.py b/python/ray/autoscaler/_private/resource_demand_scheduler.py index bb1539d63596..0154d7d3ff66 100644 --- a/python/ray/autoscaler/_private/resource_demand_scheduler.py +++ b/python/ray/autoscaler/_private/resource_demand_scheduler.py @@ -209,20 +209,10 @@ def get_nodes_to_launch( Dict of count to add for each node type, and residual of resources that still cannot be fulfilled. """ - # The following function mutates self.node_types. - self._update_node_resources_from_runtime(nodes, max_resources_by_ip) - # Does every worker node type have gpus? - # (More precisely, does every non-head node type have gpus?) - all_gpu_node_types = all( - node_type_config.get("resources").get("GPU", 0) != 0 - for node_type, node_type_config in self.node_types.items() - if node_type != self.head_node_type - ) utilization_scorer = partial( - self.utilization_scorer, - node_availability_summary=node_availability_summary, - all_gpu_node_types=all_gpu_node_types, + self.utilization_scorer, node_availability_summary=node_availability_summary ) + self._update_node_resources_from_runtime(nodes, max_resources_by_ip) node_resources: List[ResourceDict] node_type_counts: Dict[NodeType, int] @@ -816,7 +806,6 @@ def _resource_based_utilization_scorer( resources: List[ResourceDict], *, node_availability_summary: NodeAvailabilitySummary, - all_gpu_node_types: bool = False, ) -> Optional[Tuple[float, float]]: remaining = copy.deepcopy(node_resources) is_gpu_node = "GPU" in node_resources and node_resources["GPU"] > 0 @@ -824,15 +813,9 @@ def _resource_based_utilization_scorer( # Avoid launching GPU nodes if there aren't any GPU tasks at all. Note that # if there *is* a GPU task, then CPU tasks can be scheduled as well. - if not AUTOSCALER_CONSERVE_GPU_NODES: - # Skip GPU node avoidance if this is explicitly disabled. - pass - elif all_gpu_node_types: - # If ALL available non-head node types have GPUs, don't avoid upscaling. - pass - elif is_gpu_node and not any_gpu_task: - # Avoid upscaling the gpu node if the workload does not require GPUs. - return None + if AUTOSCALER_CONSERVE_GPU_NODES: + if is_gpu_node and not any_gpu_task: + return None fittable = [] resource_types = set() @@ -879,13 +862,9 @@ def _default_utilization_scorer( node_type: str, *, node_availability_summary: NodeAvailabilitySummary, - all_gpu_node_types: bool = False, ): return _resource_based_utilization_scorer( - node_resources, - resources, - node_availability_summary=node_availability_summary, - all_gpu_node_types=all_gpu_node_types, + node_resources, resources, node_availability_summary=node_availability_summary ) From ba06d1421964c2158b7dcfdb11fc56c5a33fd2e3 Mon Sep 17 00:00:00 2001 From: Dmitri Gekhtman Date: Tue, 20 Dec 2022 00:22:00 +0000 Subject: [PATCH 08/23] Implement Alex's common sense. Signed-off-by: Dmitri Gekhtman --- python/ray/autoscaler/_private/resource_demand_scheduler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/ray/autoscaler/_private/resource_demand_scheduler.py b/python/ray/autoscaler/_private/resource_demand_scheduler.py index 0154d7d3ff66..fd9fc0376ff7 100644 --- a/python/ray/autoscaler/_private/resource_demand_scheduler.py +++ b/python/ray/autoscaler/_private/resource_demand_scheduler.py @@ -815,7 +815,8 @@ def _resource_based_utilization_scorer( # if there *is* a GPU task, then CPU tasks can be scheduled as well. if AUTOSCALER_CONSERVE_GPU_NODES: if is_gpu_node and not any_gpu_task: - return None + # The lowest possible score. + return (-float("inf"), -float("inf"), -float("inf")) fittable = [] resource_types = set() From da3831ab57e3545f4f3f289888c297f472f5bf98 Mon Sep 17 00:00:00 2001 From: Dmitri Gekhtman Date: Tue, 20 Dec 2022 00:27:35 +0000 Subject: [PATCH 09/23] Fix types. Signed-off-by: Dmitri Gekhtman --- python/ray/autoscaler/_private/resource_demand_scheduler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/ray/autoscaler/_private/resource_demand_scheduler.py b/python/ray/autoscaler/_private/resource_demand_scheduler.py index fd9fc0376ff7..a047dfe97bc0 100644 --- a/python/ray/autoscaler/_private/resource_demand_scheduler.py +++ b/python/ray/autoscaler/_private/resource_demand_scheduler.py @@ -806,7 +806,7 @@ def _resource_based_utilization_scorer( resources: List[ResourceDict], *, node_availability_summary: NodeAvailabilitySummary, -) -> Optional[Tuple[float, float]]: +) -> Optional[Tuple[int, float, float]]: remaining = copy.deepcopy(node_resources) is_gpu_node = "GPU" in node_resources and node_resources["GPU"] > 0 any_gpu_task = any("GPU" in r for r in resources) @@ -816,7 +816,7 @@ def _resource_based_utilization_scorer( if AUTOSCALER_CONSERVE_GPU_NODES: if is_gpu_node and not any_gpu_task: # The lowest possible score. - return (-float("inf"), -float("inf"), -float("inf")) + return (-1, -float("inf"), -float("inf")) fittable = [] resource_types = set() From 9569cb81251d691bc0562b1d0290469d66864090 Mon Sep 17 00:00:00 2001 From: Dmitri Gekhtman Date: Tue, 20 Dec 2022 00:29:36 +0000 Subject: [PATCH 10/23] Insert reference. Signed-off-by: Dmitri Gekhtman --- python/ray/tests/test_autoscaler_fake_multinode.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/ray/tests/test_autoscaler_fake_multinode.py b/python/ray/tests/test_autoscaler_fake_multinode.py index 8b9532bdf132..bb65bd9a7e88 100644 --- a/python/ray/tests/test_autoscaler_fake_multinode.py +++ b/python/ray/tests/test_autoscaler_fake_multinode.py @@ -88,8 +88,8 @@ def ping(self): def test_autoscaler_all_gpu_node_types(): - """Validates that CPU tasks still trigger upscaling - when all available non-head node types have GPUs. + """Validates that CPU tasks can trigger GPU upscaling. + See https://github.com/ray-project/ray/pull/31202. """ gpu_node_type_1 = ( { From 9797ea436c0c3f29da3ca405d9df158c64fbf231 Mon Sep 17 00:00:00 2001 From: Dmitri Gekhtman Date: Tue, 20 Dec 2022 00:34:29 +0000 Subject: [PATCH 11/23] Fix comment. Signed-off-by: Dmitri Gekhtman --- python/ray/autoscaler/_private/resource_demand_scheduler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/ray/autoscaler/_private/resource_demand_scheduler.py b/python/ray/autoscaler/_private/resource_demand_scheduler.py index a047dfe97bc0..4c67b8ae1ab6 100644 --- a/python/ray/autoscaler/_private/resource_demand_scheduler.py +++ b/python/ray/autoscaler/_private/resource_demand_scheduler.py @@ -811,8 +811,8 @@ def _resource_based_utilization_scorer( is_gpu_node = "GPU" in node_resources and node_resources["GPU"] > 0 any_gpu_task = any("GPU" in r for r in resources) - # Avoid launching GPU nodes if there aren't any GPU tasks at all. Note that - # if there *is* a GPU task, then CPU tasks can be scheduled as well. + # Prefer not to launch a GPU node if there aren't any GPU requirements in the + # resource bundle. if AUTOSCALER_CONSERVE_GPU_NODES: if is_gpu_node and not any_gpu_task: # The lowest possible score. From 804e84372b123576c74737b44c6e23e970db901b Mon Sep 17 00:00:00 2001 From: Dmitri Gekhtman Date: Tue, 20 Dec 2022 00:36:54 +0000 Subject: [PATCH 12/23] Rearrange. Signed-off-by: Dmitri Gekhtman --- .../_private/resource_demand_scheduler.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/python/ray/autoscaler/_private/resource_demand_scheduler.py b/python/ray/autoscaler/_private/resource_demand_scheduler.py index 4c67b8ae1ab6..bde7914fdedd 100644 --- a/python/ray/autoscaler/_private/resource_demand_scheduler.py +++ b/python/ray/autoscaler/_private/resource_demand_scheduler.py @@ -808,16 +808,6 @@ def _resource_based_utilization_scorer( node_availability_summary: NodeAvailabilitySummary, ) -> Optional[Tuple[int, float, float]]: remaining = copy.deepcopy(node_resources) - is_gpu_node = "GPU" in node_resources and node_resources["GPU"] > 0 - any_gpu_task = any("GPU" in r for r in resources) - - # Prefer not to launch a GPU node if there aren't any GPU requirements in the - # resource bundle. - if AUTOSCALER_CONSERVE_GPU_NODES: - if is_gpu_node and not any_gpu_task: - # The lowest possible score. - return (-1, -float("inf"), -float("inf")) - fittable = [] resource_types = set() for r in resources: @@ -847,6 +837,15 @@ def _resource_based_utilization_scorer( if not util_by_resources: return None + # Prefer not to launch a GPU node if there aren't any GPU requirements in the + # resource bundle. + if AUTOSCALER_CONSERVE_GPU_NODES: + is_gpu_node = "GPU" in node_resources and node_resources["GPU"] > 0 + any_gpu_task = any("GPU" in r for r in resources) + if is_gpu_node and not any_gpu_task: + # The lowest possible score. + return (-1, -float("inf"), -float("inf")) + # Prioritize matching multiple resource types first, then prioritize # using all resources, then prioritize overall balance # of multiple resources. From 26ffb408737f97fdbb9f519e8030e47220c0345e Mon Sep 17 00:00:00 2001 From: Dmitri Gekhtman Date: Tue, 20 Dec 2022 00:41:03 +0000 Subject: [PATCH 13/23] Simplify test. Signed-off-by: Dmitri Gekhtman --- .../tests/test_autoscaler_fake_multinode.py | 25 +++++++------------ 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/python/ray/tests/test_autoscaler_fake_multinode.py b/python/ray/tests/test_autoscaler_fake_multinode.py index bb65bd9a7e88..f0f3500b6fc1 100644 --- a/python/ray/tests/test_autoscaler_fake_multinode.py +++ b/python/ray/tests/test_autoscaler_fake_multinode.py @@ -1,4 +1,3 @@ -import copy import pytest import platform @@ -91,24 +90,18 @@ def test_autoscaler_all_gpu_node_types(): """Validates that CPU tasks can trigger GPU upscaling. See https://github.com/ray-project/ray/pull/31202. """ - gpu_node_type_1 = ( - { - "resources": { - "CPU": 1, - "GPU": 1, - }, - "node_config": {}, - "min_workers": 0, - "max_workers": 1, - }, - ) - gpu_node_type_2 = copy.deepcopy(gpu_node_type_1) - cluster = AutoscalingCluster( head_resources={"CPU": 0}, worker_node_types={ - "gpu_node_type_1": gpu_node_type_1, - "gpu_node_type_2": gpu_node_type_2, + "gpu_node_type": { + "resources": { + "CPU": 1, + "GPU": 1, + }, + "node_config": {}, + "min_workers": 0, + "max_workers": 1, + }, }, ) From a595afaa848092b078fac139c2c8b31e34b8ad43 Mon Sep 17 00:00:00 2001 From: Dmitri Gekhtman Date: Tue, 20 Dec 2022 00:44:12 +0000 Subject: [PATCH 14/23] Explain. Signed-off-by: Dmitri Gekhtman --- python/ray/tests/test_autoscaler_fake_multinode.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/ray/tests/test_autoscaler_fake_multinode.py b/python/ray/tests/test_autoscaler_fake_multinode.py index f0f3500b6fc1..fe86d6acdf99 100644 --- a/python/ray/tests/test_autoscaler_fake_multinode.py +++ b/python/ray/tests/test_autoscaler_fake_multinode.py @@ -113,7 +113,9 @@ def test_autoscaler_all_gpu_node_types(): def task(): return True - assert ray.get(task.remote(), timeout=60), "Failed to schedule CPU task." + # Make sure the task can be scheduled. + # Since the head has 0 CPUs, this requires upscaling a GPU worker. + ray.get(task.remote(), timeout=60) ray.shutdown() finally: From 4da06da55b295831ceb1646c54a8ad6513f11346 Mon Sep 17 00:00:00 2001 From: Dmitri Gekhtman Date: Tue, 20 Dec 2022 00:44:51 +0000 Subject: [PATCH 15/23] Decrease timeout. Signed-off-by: Dmitri Gekhtman --- python/ray/tests/test_autoscaler_fake_multinode.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ray/tests/test_autoscaler_fake_multinode.py b/python/ray/tests/test_autoscaler_fake_multinode.py index fe86d6acdf99..2dba71332e69 100644 --- a/python/ray/tests/test_autoscaler_fake_multinode.py +++ b/python/ray/tests/test_autoscaler_fake_multinode.py @@ -115,7 +115,7 @@ def task(): # Make sure the task can be scheduled. # Since the head has 0 CPUs, this requires upscaling a GPU worker. - ray.get(task.remote(), timeout=60) + ray.get(task.remote(), timeout=30) ray.shutdown() finally: From a77f823389a16ae21f8df0b2c39c372da19d8213 Mon Sep 17 00:00:00 2001 From: Dmitri Gekhtman Date: Tue, 20 Dec 2022 00:45:51 +0000 Subject: [PATCH 16/23] Test name. Signed-off-by: Dmitri Gekhtman --- python/ray/tests/test_autoscaler_fake_multinode.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ray/tests/test_autoscaler_fake_multinode.py b/python/ray/tests/test_autoscaler_fake_multinode.py index 2dba71332e69..7fb8199a0588 100644 --- a/python/ray/tests/test_autoscaler_fake_multinode.py +++ b/python/ray/tests/test_autoscaler_fake_multinode.py @@ -86,7 +86,7 @@ def ping(self): cluster.shutdown() -def test_autoscaler_all_gpu_node_types(): +def test_autoscaler_cpu_task_gpu_node_up(): """Validates that CPU tasks can trigger GPU upscaling. See https://github.com/ray-project/ray/pull/31202. """ From 0e571a42894d98bb52daa10527031cd71d2633bf Mon Sep 17 00:00:00 2001 From: Dmitri Gekhtman Date: Tue, 20 Dec 2022 05:37:23 +0000 Subject: [PATCH 17/23] Prepend a field to the tuple. Signed-off-by: Dmitri Gekhtman --- .../_private/resource_demand_scheduler.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/python/ray/autoscaler/_private/resource_demand_scheduler.py b/python/ray/autoscaler/_private/resource_demand_scheduler.py index bde7914fdedd..e9be34230be8 100644 --- a/python/ray/autoscaler/_private/resource_demand_scheduler.py +++ b/python/ray/autoscaler/_private/resource_demand_scheduler.py @@ -806,7 +806,7 @@ def _resource_based_utilization_scorer( resources: List[ResourceDict], *, node_availability_summary: NodeAvailabilitySummary, -) -> Optional[Tuple[int, float, float]]: +) -> Optional[Tuple[bool, int, float, float]]: remaining = copy.deepcopy(node_resources) fittable = [] resource_types = set() @@ -839,17 +839,19 @@ def _resource_based_utilization_scorer( # Prefer not to launch a GPU node if there aren't any GPU requirements in the # resource bundle. + gpu_ok = True if AUTOSCALER_CONSERVE_GPU_NODES: is_gpu_node = "GPU" in node_resources and node_resources["GPU"] > 0 any_gpu_task = any("GPU" in r for r in resources) if is_gpu_node and not any_gpu_task: - # The lowest possible score. - return (-1, -float("inf"), -float("inf")) + gpu_ok = False - # Prioritize matching multiple resource types first, then prioritize - # using all resources, then prioritize overall balance - # of multiple resources. + # Prioritize avoiding gpu nodes for non-gpu workloads first, + # then matching multiple resource types, + # then prioritize using all resources, + # then prioritize overall balance of multiple resources. return ( + gpu_ok, num_matching_resource_types, min(util_by_resources), np.mean(util_by_resources), From 7f5056779031ec382c9a4521078e90f7cb9cace6 Mon Sep 17 00:00:00 2001 From: Dmitri Gekhtman Date: Tue, 20 Dec 2022 05:53:29 +0000 Subject: [PATCH 18/23] Tests. Signed-off-by: Dmitri Gekhtman --- .../tests/test_resource_demand_scheduler.py | 44 ++++++++----------- 1 file changed, 18 insertions(+), 26 deletions(-) diff --git a/python/ray/tests/test_resource_demand_scheduler.py b/python/ray/tests/test_resource_demand_scheduler.py index 61ec7e56cb68..4259dacf6a88 100644 --- a/python/ray/tests/test_resource_demand_scheduler.py +++ b/python/ray/tests/test_resource_demand_scheduler.py @@ -98,71 +98,63 @@ def test_util_score(): {"GPU": 4}, [{"GPU": 1}, {"GPU": 1}], node_availability_summary=EMPTY_AVAILABILITY_SUMMARY, - ) == (1, 0.5, 0.5) + ) == (True, 1, 0.5, 0.5) assert _resource_based_utilization_scorer( {"GPU": 2}, [{"GPU": 2}], node_availability_summary=EMPTY_AVAILABILITY_SUMMARY - ) == (1, 2, 2) + ) == (True, 1, 2, 2) assert _resource_based_utilization_scorer( {"GPU": 2}, [{"GPU": 1}, {"GPU": 1}], node_availability_summary=EMPTY_AVAILABILITY_SUMMARY, - ) == (1, 2, 2) + ) == (True, 1, 2, 2) assert _resource_based_utilization_scorer( {"GPU": 1}, [{"GPU": 1, "CPU": 1}, {"GPU": 1}], node_availability_summary=EMPTY_AVAILABILITY_SUMMARY, - ) == ( - 1, - 1, - 1, - ) + ) == (True, 1, 1, 1) assert _resource_based_utilization_scorer( {"GPU": 1, "CPU": 1}, [{"GPU": 1, "CPU": 1}, {"GPU": 1}], node_availability_summary=EMPTY_AVAILABILITY_SUMMARY, - ) == (2, 1, 1) + ) == (True, 2, 1, 1) assert _resource_based_utilization_scorer( {"GPU": 2, "TPU": 1}, [{"GPU": 2}], node_availability_summary=EMPTY_AVAILABILITY_SUMMARY, - ) == (1, 0, 1) + ) == (True, 1, 0, 1) assert _resource_based_utilization_scorer( {"CPU": 64}, [{"CPU": 64}], node_availability_summary=EMPTY_AVAILABILITY_SUMMARY - ) == (1, 64, 64) + ) == (True, 1, 64, 64) assert _resource_based_utilization_scorer( {"CPU": 64}, [{"CPU": 32}], node_availability_summary=EMPTY_AVAILABILITY_SUMMARY - ) == (1, 8, 8) + ) == (True, 1, 8, 8) assert _resource_based_utilization_scorer( {"CPU": 64}, [{"CPU": 16}, {"CPU": 16}], node_availability_summary=EMPTY_AVAILABILITY_SUMMARY, - ) == (1, 8, 8) + ) == (True, 1, 8, 8) def test_gpu_node_util_score(): # Avoid scheduling CPU tasks on GPU node. - assert ( - _resource_based_utilization_scorer( - {"GPU": 1, "CPU": 1}, - [{"CPU": 1}], - node_availability_summary=EMPTY_AVAILABILITY_SUMMARY, - ) - is None + utilization_score = _resource_based_utilization_scorer( + {"GPU": 1, "CPU": 1}, + [{"CPU": 1}], + node_availability_summary=EMPTY_AVAILABILITY_SUMMARY, ) + import pdb; pdb.set_trace() + gpu_ok = utilization_score[0] + assert gpu_ok is False assert _resource_based_utilization_scorer( {"GPU": 1, "CPU": 1}, [{"CPU": 1, "GPU": 1}], node_availability_summary=EMPTY_AVAILABILITY_SUMMARY, - ) == ( - 2, - 1.0, - 1.0, - ) + ) == (True, 2, 1.0, 1.0) assert _resource_based_utilization_scorer( {"GPU": 1, "CPU": 1}, [{"GPU": 1}], node_availability_summary=EMPTY_AVAILABILITY_SUMMARY, - ) == (1, 0.0, 0.5) + ) == (True, 1, 0.0, 0.5) def test_zero_resource(): From e0bc69d665c6feb5dfc8cd0ee00375570a5dc94f Mon Sep 17 00:00:00 2001 From: Dmitri Gekhtman Date: Tue, 20 Dec 2022 05:53:51 +0000 Subject: [PATCH 19/23] lint Signed-off-by: Dmitri Gekhtman --- python/ray/tests/test_resource_demand_scheduler.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/ray/tests/test_resource_demand_scheduler.py b/python/ray/tests/test_resource_demand_scheduler.py index 4259dacf6a88..13ea8555c478 100644 --- a/python/ray/tests/test_resource_demand_scheduler.py +++ b/python/ray/tests/test_resource_demand_scheduler.py @@ -142,7 +142,9 @@ def test_gpu_node_util_score(): [{"CPU": 1}], node_availability_summary=EMPTY_AVAILABILITY_SUMMARY, ) - import pdb; pdb.set_trace() + import pdb + + pdb.set_trace() gpu_ok = utilization_score[0] assert gpu_ok is False assert _resource_based_utilization_scorer( From 27ffedd7c537256ffc3ea4746e6376914e87a8ae Mon Sep 17 00:00:00 2001 From: Dmitri Gekhtman Date: Tue, 20 Dec 2022 06:00:34 +0000 Subject: [PATCH 20/23] remove breakpoint Signed-off-by: Dmitri Gekhtman --- python/ray/tests/test_resource_demand_scheduler.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/python/ray/tests/test_resource_demand_scheduler.py b/python/ray/tests/test_resource_demand_scheduler.py index 13ea8555c478..4fc7642b1311 100644 --- a/python/ray/tests/test_resource_demand_scheduler.py +++ b/python/ray/tests/test_resource_demand_scheduler.py @@ -142,9 +142,6 @@ def test_gpu_node_util_score(): [{"CPU": 1}], node_availability_summary=EMPTY_AVAILABILITY_SUMMARY, ) - import pdb - - pdb.set_trace() gpu_ok = utilization_score[0] assert gpu_ok is False assert _resource_based_utilization_scorer( From b812676176c3269cd0c532aa7d439bdede374853 Mon Sep 17 00:00:00 2001 From: Dmitri Gekhtman Date: Tue, 20 Dec 2022 06:24:46 +0000 Subject: [PATCH 21/23] Fix test. Signed-off-by: Dmitri Gekhtman --- python/ray/tests/test_resource_demand_scheduler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ray/tests/test_resource_demand_scheduler.py b/python/ray/tests/test_resource_demand_scheduler.py index 4fc7642b1311..6e255d7c729d 100644 --- a/python/ray/tests/test_resource_demand_scheduler.py +++ b/python/ray/tests/test_resource_demand_scheduler.py @@ -93,7 +93,7 @@ def test_util_score(): ) assert _resource_based_utilization_scorer( {"GPU": 4}, [{"GPU": 2}], node_availability_summary=EMPTY_AVAILABILITY_SUMMARY - ) == (1, 0.5, 0.5) + ) == (True, 1, 0.5, 0.5) assert _resource_based_utilization_scorer( {"GPU": 4}, [{"GPU": 1}, {"GPU": 1}], From 83c6e1591f9005b15250fc1523da333a81c2f4db Mon Sep 17 00:00:00 2001 From: Dmitri Gekhtman Date: Tue, 20 Dec 2022 06:41:17 +0000 Subject: [PATCH 22/23] Add to test. Signed-off-by: Dmitri Gekhtman --- python/ray/tests/test_resource_demand_scheduler.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/python/ray/tests/test_resource_demand_scheduler.py b/python/ray/tests/test_resource_demand_scheduler.py index 6e255d7c729d..f575ab359a31 100644 --- a/python/ray/tests/test_resource_demand_scheduler.py +++ b/python/ray/tests/test_resource_demand_scheduler.py @@ -313,13 +313,23 @@ def test_gpu_node_avoid_cpu_task(): }, } r1 = [{"CPU": 1}] * 100 + # max_to_add ten nodes allowed. All chosen to be "cpu". assert get_nodes_for( types, {}, "empty_node", - 100, + 10, r1, ) == {"cpu": 10} + # max_to_add eleven nodes allowed. First ten chosen to be "cpu", + # last chosen to be "gpu" due max_workers constraint on "cpu". + assert get_nodes_for( + types, + {}, + "empty_node", + 11, + r1, + ) == {"cpu": 10, "gpu": 1} r2 = [{"GPU": 1}] + [{"CPU": 1}] * 100 assert get_nodes_for( types, From d151ac80d168a4a9e1c21a2aa8f2b25a9cdcdc70 Mon Sep 17 00:00:00 2001 From: Dmitri Gekhtman Date: Tue, 20 Dec 2022 23:52:10 +0000 Subject: [PATCH 23/23] Comment wording. Signed-off-by: Dmitri Gekhtman --- python/ray/autoscaler/_private/resource_demand_scheduler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ray/autoscaler/_private/resource_demand_scheduler.py b/python/ray/autoscaler/_private/resource_demand_scheduler.py index e9be34230be8..f39a8830a75a 100644 --- a/python/ray/autoscaler/_private/resource_demand_scheduler.py +++ b/python/ray/autoscaler/_private/resource_demand_scheduler.py @@ -847,7 +847,7 @@ def _resource_based_utilization_scorer( gpu_ok = False # Prioritize avoiding gpu nodes for non-gpu workloads first, - # then matching multiple resource types, + # then prioritize matching multiple resource types, # then prioritize using all resources, # then prioritize overall balance of multiple resources. return (