From a6ca7bb7ec0be8a3a6911baab0fdc1dabbcba7e3 Mon Sep 17 00:00:00 2001 From: kaihsun Date: Tue, 8 Nov 2022 07:40:18 +0000 Subject: [PATCH 1/6] sample YAMLs --- .../config/samples/ray-cluster.autoscaler.large.yaml | 3 --- ray-operator/config/samples/ray-cluster.autoscaler.yaml | 3 --- .../config/samples/ray-cluster.complete.large.yaml | 2 -- ray-operator/config/samples/ray-cluster.complete.yaml | 2 -- .../config/samples/ray-cluster.external-redis.yaml | 2 -- .../config/samples/ray-cluster.getting-started.yaml | 4 ---- .../config/samples/ray-cluster.heterogeneous.yaml | 6 ------ ray-operator/config/samples/ray-cluster.mini.yaml | 4 ---- ray-operator/config/samples/ray_v1alpha1_rayjob.yaml | 4 ---- ray-operator/config/samples/ray_v1alpha1_rayservice.yaml | 8 ++++---- 10 files changed, 4 insertions(+), 34 deletions(-) diff --git a/ray-operator/config/samples/ray-cluster.autoscaler.large.yaml b/ray-operator/config/samples/ray-cluster.autoscaler.large.yaml index b874c68418..d9dafe2449 100644 --- a/ray-operator/config/samples/ray-cluster.autoscaler.large.yaml +++ b/ray-operator/config/samples/ray-cluster.autoscaler.large.yaml @@ -56,9 +56,6 @@ spec: headGroupSpec: # Kubernetes Service Type, valid values are 'ClusterIP', 'NodePort' and 'LoadBalancer' serviceType: ClusterIP - # logical group name, for this called head-group, also can be functional - # pod type head or worker - # rayNodeType: head # Not needed since it is under the headgroup # the following params are used to complete the ray start: ray start --head --block --port=6379 ... rayStartParams: # Flag "no-monitor" will be automatically set when autoscaling is enabled. diff --git a/ray-operator/config/samples/ray-cluster.autoscaler.yaml b/ray-operator/config/samples/ray-cluster.autoscaler.yaml index cd21639785..11c290000d 100644 --- a/ray-operator/config/samples/ray-cluster.autoscaler.yaml +++ b/ray-operator/config/samples/ray-cluster.autoscaler.yaml @@ -47,9 +47,6 @@ spec: headGroupSpec: # Kubernetes Service Type, valid values are 'ClusterIP', 'NodePort' and 'LoadBalancer' serviceType: ClusterIP - # logical group name, for this called head-group, also can be functional - # pod type head or worker - # rayNodeType: head # Not needed since it is under the headgroup # the following params are used to complete the ray start: ray start --head --block ... rayStartParams: # Flag "no-monitor" will be automatically set when autoscaling is enabled. diff --git a/ray-operator/config/samples/ray-cluster.complete.large.yaml b/ray-operator/config/samples/ray-cluster.complete.large.yaml index 2944a6e571..dd39511981 100644 --- a/ray-operator/config/samples/ray-cluster.complete.large.yaml +++ b/ray-operator/config/samples/ray-cluster.complete.large.yaml @@ -33,7 +33,6 @@ spec: # custom labels. NOTE: do not define custom labels start with `raycluster.`, they may be used in controller. # Refer to https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ rayCluster: raycluster-sample # will be injected if missing - rayNodeType: head # will be injected if missing, must be head or wroker groupName: headgroup # will be injected if missing # annotations for pod annotations: @@ -94,7 +93,6 @@ spec: metadata: labels: rayCluster: raycluster-complete # will be injected if missing - rayNodeType: worker # will be injected if missing groupName: small-group # will be injected if missing spec: containers: diff --git a/ray-operator/config/samples/ray-cluster.complete.yaml b/ray-operator/config/samples/ray-cluster.complete.yaml index 5042a40a8b..7c973ed139 100644 --- a/ray-operator/config/samples/ray-cluster.complete.yaml +++ b/ray-operator/config/samples/ray-cluster.complete.yaml @@ -30,7 +30,6 @@ spec: # custom labels. NOTE: do not define custom labels start with `raycluster.`, they may be used in controller. # Refer to https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ rayCluster: raycluster-sample # will be injected if missing - rayNodeType: head # will be injected if missing, must be head or wroker groupName: headgroup # will be injected if missing # annotations for pod annotations: @@ -87,7 +86,6 @@ spec: metadata: labels: rayCluster: raycluster-complete # will be injected if missing - rayNodeType: worker # will be injected if missing groupName: small-group # will be injected if missing spec: containers: diff --git a/ray-operator/config/samples/ray-cluster.external-redis.yaml b/ray-operator/config/samples/ray-cluster.external-redis.yaml index 6337a5a34a..970b3c44ed 100644 --- a/ray-operator/config/samples/ray-cluster.external-redis.yaml +++ b/ray-operator/config/samples/ray-cluster.external-redis.yaml @@ -93,7 +93,6 @@ spec: # custom labels. NOTE: do not define custom labels start with `raycluster.`, they may be used in controller. # Refer to https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ rayCluster: raycluster-external-redis # will be injected if missing - rayNodeType: head # will be injected if missing, must be head or wroker groupName: headgroup # will be injected if missing spec: containers: @@ -135,7 +134,6 @@ spec: metadata: labels: rayCluster: raycluster-external-redis # will be injected if missing - rayNodeType: worker # will be injected if missing groupName: small-group # will be injected if missing # annotations for pod annotations: diff --git a/ray-operator/config/samples/ray-cluster.getting-started.yaml b/ray-operator/config/samples/ray-cluster.getting-started.yaml index 02784ad45b..8effc1fde2 100644 --- a/ray-operator/config/samples/ray-cluster.getting-started.yaml +++ b/ray-operator/config/samples/ray-cluster.getting-started.yaml @@ -18,9 +18,6 @@ spec: serviceType: ClusterIP # the pod replicas in this group typed head (assuming there could be more than 1 in the future) replicas: 1 - # logical group name, for this called head-group, also can be functional - # pod type head or worker - # rayNodeType: head # Not needed since it is under the headgroup # the following params are used to complete the ray start: ray start --head --block --redis-port=6379 ... rayStartParams: port: '6379' # should match headService targetPort @@ -37,7 +34,6 @@ spec: # custom labels. NOTE: do not define custom labels start with `raycluster.`, they may be used in controller. # Refer to https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ rayCluster: raycluster-sample # will be injected if missing - rayNodeType: head # will be injected if missing, must be head or wroker groupName: headgroup # will be injected if missing # annotations for pod annotations: diff --git a/ray-operator/config/samples/ray-cluster.heterogeneous.yaml b/ray-operator/config/samples/ray-cluster.heterogeneous.yaml index 6a651eecbe..d4312f92e4 100644 --- a/ray-operator/config/samples/ray-cluster.heterogeneous.yaml +++ b/ray-operator/config/samples/ray-cluster.heterogeneous.yaml @@ -44,9 +44,6 @@ spec: serviceType: ClusterIP # the pod replicas in this group typed head (assuming there could be more than 1 in the future) replicas: 1 - # logical group name, for this called head-group, also can be functional - # pod type head or worker - # rayNodeType: head # Not needed since it is under the headgroup # the following params are used to complete the ray start: ray start --head --block --redis-port=6379 ... rayStartParams: port: '6379' @@ -63,7 +60,6 @@ spec: # custom labels. NOTE: do not define custom labels start with `raycluster.`, they may be used in controller. # Refer to https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ rayCluster: raycluster-heterogeneous # will be injected if missing - rayNodeType: head # will be injected if missing, must be head or wroker groupName: headgroup # will be injected if missing # annotations for pod annotations: @@ -125,7 +121,6 @@ spec: metadata: labels: rayCluster: raycluster-heterogeneous # will be injected if missing - rayNodeType: worker # will be injected if missing groupName: small-group # will be injected if missing # annotations for pod annotations: @@ -200,7 +195,6 @@ spec: metadata: labels: rayCluster: raycluster-heterogeneous # will be injected if missing - rayNodeType: worker # will be injected if missing # annotations for pod annotations: key: value diff --git a/ray-operator/config/samples/ray-cluster.mini.yaml b/ray-operator/config/samples/ray-cluster.mini.yaml index c297f9f0da..e14d88e446 100644 --- a/ray-operator/config/samples/ray-cluster.mini.yaml +++ b/ray-operator/config/samples/ray-cluster.mini.yaml @@ -18,9 +18,6 @@ spec: serviceType: ClusterIP # the pod replicas in this group typed head (assuming there could be more than 1 in the future) replicas: 1 - # logical group name, for this called head-group, also can be functional - # pod type head or worker - # rayNodeType: head # Not needed since it is under the headgroup # the following params are used to complete the ray start: ray start --head --block --redis-port=6379 ... rayStartParams: port: '6379' # should match container port named gcs-server @@ -38,7 +35,6 @@ spec: # custom labels. NOTE: do not define custom labels start with `raycluster.`, they may be used in controller. # Refer to https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ rayCluster: raycluster-sample # will be injected if missing - rayNodeType: head # will be injected if missing, must be head or wroker groupName: headgroup # will be injected if missing # annotations for pod annotations: diff --git a/ray-operator/config/samples/ray_v1alpha1_rayjob.yaml b/ray-operator/config/samples/ray_v1alpha1_rayjob.yaml index b82473c949..7c1d4d17f7 100644 --- a/ray-operator/config/samples/ray_v1alpha1_rayjob.yaml +++ b/ray-operator/config/samples/ray_v1alpha1_rayjob.yaml @@ -23,9 +23,6 @@ spec: serviceType: ClusterIP # the pod replicas in this group typed head (assuming there could be more than 1 in the future) replicas: 1 - # logical group name, for this called head-group, also can be functional - # pod type head or worker - # rayNodeType: head # Not needed since it is under the headgroup # the following params are used to complete the ray start: ray start --head --block --redis-port=6379 ... rayStartParams: port: '6379' # should match container port named gcs-server @@ -40,7 +37,6 @@ spec: # custom labels. NOTE: do not define custom labels start with `raycluster.`, they may be used in controller. # Refer to https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ rayCluster: raycluster-sample # will be injected if missing - rayNodeType: head # will be injected if missing, must be head or wroker groupName: headgroup # will be injected if missing # annotations for pod annotations: diff --git a/ray-operator/config/samples/ray_v1alpha1_rayservice.yaml b/ray-operator/config/samples/ray_v1alpha1_rayservice.yaml index 85c0e72707..60db9302a0 100644 --- a/ray-operator/config/samples/ray_v1alpha1_rayservice.yaml +++ b/ray-operator/config/samples/ray_v1alpha1_rayservice.yaml @@ -50,9 +50,6 @@ spec: serviceType: ClusterIP # the pod replicas in this group typed head (assuming there could be more than 1 in the future) replicas: 1 - # logical group name, for this called head-group, also can be functional - # pod type head or worker - # rayNodeType: head # Not needed since it is under the headgroup # the following params are used to complete the ray start: ray start --head --block --redis-port=6379 ... rayStartParams: port: '6379' # should match container port named gcs-server @@ -70,7 +67,6 @@ spec: # custom labels. NOTE: do not define custom labels start with `raycluster.`, they may be used in controller. # Refer to https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ rayCluster: raycluster-sample # will be injected if missing - rayNodeType: head # will be injected if missing, must be head or wroker groupName: headgroup # will be injected if missing # annotations for pod annotations: @@ -86,6 +82,8 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP + - name: SERVE_DEPLOYMENT_HANDLE_IS_SYNC + value: "1" resources: limits: cpu: 2 @@ -175,6 +173,8 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP + - name: SERVE_DEPLOYMENT_HANDLE_IS_SYNC + value: "1" ports: - containerPort: 80 name: client From 85a457ab1cb14e169e52c434ab6ff3fab9c57d44 Mon Sep 17 00:00:00 2001 From: kaihsun Date: Tue, 8 Nov 2022 07:46:54 +0000 Subject: [PATCH 2/6] pass go unit tests --- ray-operator/apis/ray/v1alpha1/rayjob_types_test.go | 8 +++----- ray-operator/apis/ray/v1alpha1/rayservice_types_test.go | 8 +++----- ray-operator/controllers/ray/rayjob_controller_test.go | 5 ++--- .../controllers/ray/rayservice_controller_test.go | 5 ++--- 4 files changed, 10 insertions(+), 16 deletions(-) diff --git a/ray-operator/apis/ray/v1alpha1/rayjob_types_test.go b/ray-operator/apis/ray/v1alpha1/rayjob_types_test.go index a6cbbfc5d9..73678801d5 100644 --- a/ray-operator/apis/ray/v1alpha1/rayjob_types_test.go +++ b/ray-operator/apis/ray/v1alpha1/rayjob_types_test.go @@ -38,9 +38,8 @@ var expectedRayJob = RayJob{ Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ - "rayCluster": "raycluster-sample", - "rayNodeType": "head", - "groupName": "headgroup", + "rayCluster": "raycluster-sample", + "groupName": "headgroup", }, Annotations: map[string]string{ "key": "value", @@ -166,8 +165,7 @@ var testRayJobJSON = `{ "creationTimestamp": null, "labels": { "groupName": "headgroup", - "rayCluster": "raycluster-sample", - "rayNodeType": "head" + "rayCluster": "raycluster-sample" }, "annotations": { "key": "value" diff --git a/ray-operator/apis/ray/v1alpha1/rayservice_types_test.go b/ray-operator/apis/ray/v1alpha1/rayservice_types_test.go index c3bf5d3290..d2997d4651 100644 --- a/ray-operator/apis/ray/v1alpha1/rayservice_types_test.go +++ b/ray-operator/apis/ray/v1alpha1/rayservice_types_test.go @@ -86,9 +86,8 @@ var myRayService = &RayService{ Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ - "rayCluster": "raycluster-sample", - "rayNodeType": "head", - "groupName": "headgroup", + "rayCluster": "raycluster-sample", + "groupName": "headgroup", }, Annotations: map[string]string{ "key": "value", @@ -258,8 +257,7 @@ var expected = `{ "creationTimestamp":null, "labels":{ "groupName":"headgroup", - "rayCluster":"raycluster-sample", - "rayNodeType":"head" + "rayCluster":"raycluster-sample" }, "annotations":{ "key":"value" diff --git a/ray-operator/controllers/ray/rayjob_controller_test.go b/ray-operator/controllers/ray/rayjob_controller_test.go index 2d325cd6d5..316fd4785e 100644 --- a/ray-operator/controllers/ray/rayjob_controller_test.go +++ b/ray-operator/controllers/ray/rayjob_controller_test.go @@ -65,9 +65,8 @@ var _ = Context("Inside the default namespace", func() { Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ - "rayCluster": "raycluster-sample", - "rayNodeType": "head", - "groupName": "headgroup", + "rayCluster": "raycluster-sample", + "groupName": "headgroup", }, Annotations: map[string]string{ "key": "value", diff --git a/ray-operator/controllers/ray/rayservice_controller_test.go b/ray-operator/controllers/ray/rayservice_controller_test.go index 98801cf955..2ff0be1316 100644 --- a/ray-operator/controllers/ray/rayservice_controller_test.go +++ b/ray-operator/controllers/ray/rayservice_controller_test.go @@ -102,9 +102,8 @@ var _ = Context("Inside the default namespace", func() { Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ - "rayCluster": "raycluster-sample", - "rayNodeType": "head", - "groupName": "headgroup", + "rayCluster": "raycluster-sample", + "groupName": "headgroup", }, Annotations: map[string]string{ "key": "value", From 76aadaeeb26200c0099b57d5a591c5a2906ebd29 Mon Sep 17 00:00:00 2001 From: kaihsun Date: Tue, 8 Nov 2022 07:50:13 +0000 Subject: [PATCH 3/6] prometheus --- config/prometheus/podMonitor.yaml | 1 - docs/guidance/observability.md | 1 - 2 files changed, 2 deletions(-) diff --git a/config/prometheus/podMonitor.yaml b/config/prometheus/podMonitor.yaml index fb92ab5a52..316a0337dd 100644 --- a/config/prometheus/podMonitor.yaml +++ b/config/prometheus/podMonitor.yaml @@ -11,7 +11,6 @@ spec: - ray-system selector: matchLabels: - rayNodeType: worker ray.io/node-type: worker ray.io/is-ray-node: "yes" podMetricsEndpoints: diff --git a/docs/guidance/observability.md b/docs/guidance/observability.md index 0ed25569e1..246391bbd9 100644 --- a/docs/guidance/observability.md +++ b/docs/guidance/observability.md @@ -265,7 +265,6 @@ spec: matchLabels: ray.io/is-ray-node: "yes" ray.io/node-type: worker - rayNodeType: worker ``` Since we are not selecting a Kubernetes service but pods, our `matchLabels` now define a set of labels that is common on all Ray workers. From 13443f5002a9e875b8103f23583c0059a3d56411 Mon Sep 17 00:00:00 2001 From: kaihsun Date: Tue, 8 Nov 2022 09:08:56 +0000 Subject: [PATCH 4/6] compatibility test & config test --- tests/compatibility-test.py | 4 ++-- tests/config/ray-cluster.mini.yaml.template | 5 ----- tests/config/ray-cluster.ray-ft.yaml.template | 2 -- tests/config/ray-service-cluster-update.yaml.template | 4 ---- tests/config/ray-service-serve-update.yaml.template | 4 ---- tests/config/ray-service.yaml.template | 4 ---- tests/framework/config/ray-cluster.mini.yaml.template | 5 ----- tests/kuberay_utils/utils.py | 2 +- 8 files changed, 3 insertions(+), 27 deletions(-) diff --git a/tests/compatibility-test.py b/tests/compatibility-test.py index b59188398a..fd825302d7 100755 --- a/tests/compatibility-test.py +++ b/tests/compatibility-test.py @@ -194,7 +194,7 @@ def test_ray_serve(self): # KubeRay only allows at most 1 head pod per RayCluster instance at the same time. In addition, # if we have 0 head pods at this moment, it indicates that the head pod crashes unexpectedly. - headpods = utils.get_pod(k8s_api, namespace='default', label_selector='rayNodeType=head') + headpods = utils.get_pod(k8s_api, namespace='default', label_selector='ray.io/node-type=head') assert(len(headpods.items) == 1) old_head_pod = headpods.items[0] old_head_pod_name = old_head_pod.metadata.name @@ -240,7 +240,7 @@ def test_detached_actor(self): # KubeRay only allows at most 1 head pod per RayCluster instance at the same time. In addition, # if we have 0 head pods at this moment, it indicates that the head pod crashes unexpectedly. - headpods = utils.get_pod(k8s_api, namespace='default', label_selector='rayNodeType=head') + headpods = utils.get_pod(k8s_api, namespace='default', label_selector='ray.io/node-type=head') assert(len(headpods.items) == 1) old_head_pod = headpods.items[0] old_head_pod_name = old_head_pod.metadata.name diff --git a/tests/config/ray-cluster.mini.yaml.template b/tests/config/ray-cluster.mini.yaml.template index 61324b40ba..e9fc65d8b8 100644 --- a/tests/config/ray-cluster.mini.yaml.template +++ b/tests/config/ray-cluster.mini.yaml.template @@ -14,9 +14,6 @@ spec: serviceType: ClusterIP # the pod replicas in this group typed head (assuming there could be more than 1 in the future) replicas: 1 - # logical group name, for this called head-group, also can be functional - # pod type head or worker - # rayNodeType: head # Not needed since it is under the headgroup # the following params are used to complete the ray start: ray start --head --block --redis-port=6379 ... rayStartParams: port: '6379' # should match headService targetPort @@ -34,7 +31,6 @@ spec: # custom labels. NOTE: do not define custom labels start with `raycluster.`, they may be used in controller. # Refer to https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ rayCluster: raycluster-compatibility-test - rayNodeType: head # will be injected if missing, must be head or wroker groupName: headgroup # will be injected if missing # annotations for pod annotations: @@ -70,7 +66,6 @@ spec: metadata: labels: rayCluster: raycluster-compatibility-test - rayNodeType: worker # will be injected if missing groupName: small-group # will be injected if missing # annotations for pod annotations: diff --git a/tests/config/ray-cluster.ray-ft.yaml.template b/tests/config/ray-cluster.ray-ft.yaml.template index d69d699fef..37203da4cc 100644 --- a/tests/config/ray-cluster.ray-ft.yaml.template +++ b/tests/config/ray-cluster.ray-ft.yaml.template @@ -85,7 +85,6 @@ spec: metadata: labels: rayCluster: raycluster-compatibility-test - rayNodeType: head groupName: headgroup # will be injected if missing spec: containers: @@ -131,7 +130,6 @@ spec: metadata: labels: rayCluster: raycluster-compatibility-test - rayNodeType: worker # will be injected if missing groupName: small-group # will be injected if missing # annotations for pod annotations: diff --git a/tests/config/ray-service-cluster-update.yaml.template b/tests/config/ray-service-cluster-update.yaml.template index 63e936d1c7..e3f1b7395a 100644 --- a/tests/config/ray-service-cluster-update.yaml.template +++ b/tests/config/ray-service-cluster-update.yaml.template @@ -46,9 +46,6 @@ spec: serviceType: ClusterIP # the pod replicas in this group typed head (assuming there could be more than 1 in the future) replicas: 1 - # logical group name, for this called head-group, also can be functional - # pod type head or worker - # rayNodeType: head # Not needed since it is under the headgroup # the following params are used to complete the ray start: ray start --head --block --redis-port=6379 ... rayStartParams: port: '6379' # should match container port named gcs-server @@ -67,7 +64,6 @@ spec: # custom labels. NOTE: do not define custom labels start with `raycluster.`, they may be used in controller. # Refer to https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ rayCluster: raycluster-compatibility-test # will be injected if missing - rayNodeType: head # will be injected if missing, must be head or wroker groupName: headgroup # will be injected if missing # annotations for pod annotations: diff --git a/tests/config/ray-service-serve-update.yaml.template b/tests/config/ray-service-serve-update.yaml.template index b52b8ce753..d70c7cf15e 100644 --- a/tests/config/ray-service-serve-update.yaml.template +++ b/tests/config/ray-service-serve-update.yaml.template @@ -46,9 +46,6 @@ spec: serviceType: ClusterIP # the pod replicas in this group typed head (assuming there could be more than 1 in the future) replicas: 1 - # logical group name, for this called head-group, also can be functional - # pod type head or worker - # rayNodeType: head # Not needed since it is under the headgroup # the following params are used to complete the ray start: ray start --head --block --redis-port=6379 ... rayStartParams: port: '6379' # should match container port named gcs-server @@ -67,7 +64,6 @@ spec: # custom labels. NOTE: do not define custom labels start with `raycluster.`, they may be used in controller. # Refer to https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ rayCluster: raycluster-compatibility-test # will be injected if missing - rayNodeType: head # will be injected if missing, must be head or wroker groupName: headgroup # will be injected if missing # annotations for pod annotations: diff --git a/tests/config/ray-service.yaml.template b/tests/config/ray-service.yaml.template index cc3be134e5..9deee3492a 100644 --- a/tests/config/ray-service.yaml.template +++ b/tests/config/ray-service.yaml.template @@ -46,9 +46,6 @@ spec: serviceType: ClusterIP # the pod replicas in this group typed head (assuming there could be more than 1 in the future) replicas: 1 - # logical group name, for this called head-group, also can be functional - # pod type head or worker - # rayNodeType: head # Not needed since it is under the headgroup # the following params are used to complete the ray start: ray start --head --block --redis-port=6379 ... rayStartParams: port: '6379' # should match container port named gcs-server @@ -67,7 +64,6 @@ spec: # custom labels. NOTE: do not define custom labels start with `raycluster.`, they may be used in controller. # Refer to https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ rayCluster: raycluster-compatibility-test # will be injected if missing - rayNodeType: head # will be injected if missing, must be head or wroker groupName: headgroup # will be injected if missing # annotations for pod annotations: diff --git a/tests/framework/config/ray-cluster.mini.yaml.template b/tests/framework/config/ray-cluster.mini.yaml.template index ff4075b50e..4aece7106f 100644 --- a/tests/framework/config/ray-cluster.mini.yaml.template +++ b/tests/framework/config/ray-cluster.mini.yaml.template @@ -14,9 +14,6 @@ spec: serviceType: ClusterIP # the pod replicas in this group typed head (assuming there could be more than 1 in the future) replicas: 1 - # logical group name, for this called head-group, also can be functional - # pod type head or worker - # rayNodeType: head # Not needed since it is under the headgroup # the following params are used to complete the ray start: ray start --head --block --redis-port=6379 ... rayStartParams: port: '6379' # should match headService targetPort @@ -34,7 +31,6 @@ spec: # custom labels. NOTE: do not define custom labels start with `raycluster.`, they may be used in controller. # Refer to https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ rayCluster: raycluster-compatibility-test - rayNodeType: head # will be injected if missing, must be head or wroker groupName: headgroup # will be injected if missing # annotations for pod annotations: @@ -77,7 +73,6 @@ spec: metadata: labels: rayCluster: raycluster-compatibility-test - rayNodeType: worker # will be injected if missing groupName: small-group # will be injected if missing # annotations for pod annotations: diff --git a/tests/kuberay_utils/utils.py b/tests/kuberay_utils/utils.py index 82a67a746b..0bbc743ee0 100644 --- a/tests/kuberay_utils/utils.py +++ b/tests/kuberay_utils/utils.py @@ -260,7 +260,7 @@ def wait_for_new_head(k8s_api, old_head_pod_name, old_restart_count, namespace, """ def check_status(k8s_api, old_head_pod_name, old_restart_count, namespace) -> bool: all_pods = k8s_api.list_namespaced_pod(namespace = namespace) - headpods = get_pod(k8s_api, namespace=namespace, label_selector='rayNodeType=head') + headpods = get_pod(k8s_api, namespace=namespace, label_selector='ray.io/node-type=head') # KubeRay only allows at most 1 head pod per RayCluster instance at the same time. On the other # hands, when we kill a worker, the operator will reconcile a new one immediately without waiting # for the Pod termination to complete. Hence, it is possible to have more than `worker.Replicas` From e150a65a2ff46013a4d0a63c92094c82663f1fa7 Mon Sep 17 00:00:00 2001 From: kaihsun Date: Tue, 8 Nov 2022 09:11:04 +0000 Subject: [PATCH 5/6] helm chart --- helm-chart/ray-cluster/templates/raycluster-cluster.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/helm-chart/ray-cluster/templates/raycluster-cluster.yaml b/helm-chart/ray-cluster/templates/raycluster-cluster.yaml index 228f241e75..d4d5a2b711 100644 --- a/helm-chart/ray-cluster/templates/raycluster-cluster.yaml +++ b/helm-chart/ray-cluster/templates/raycluster-cluster.yaml @@ -58,7 +58,6 @@ spec: annotations: {{- toYaml .Values.head.annotations | nindent 10 }} labels: groupName: {{ .Values.head.groupName }} - rayNodeType: {{ .Values.head.type }} rayCluster: {{ include "ray-cluster.fullname" . }} {{ include "ray-cluster.labels" . | indent 10 }} @@ -108,7 +107,6 @@ spec: metadata: annotations: {{- toYaml $values.annotations | nindent 10 }} labels: - rayNodeType: {{ $values.type }} groupName: {{ $groupName }} rayCluster: {{ include "ray-cluster.fullname" $ }} {{ include "ray-cluster.labels" $ | indent 10 }} @@ -158,7 +156,6 @@ spec: metadata: annotations: {{- toYaml .Values.worker.annotations | nindent 10 }} labels: - rayNodeType: {{ .Values.worker.type }} groupName: {{ .Values.worker.groupName }} rayCluster: {{ include "ray-cluster.fullname" . }} {{ include "ray-cluster.labels" . | indent 10 }} From 16b690ca7074826af5221610584791ec172429d3 Mon Sep 17 00:00:00 2001 From: kaihsun Date: Wed, 9 Nov 2022 00:46:47 +0000 Subject: [PATCH 6/6] update service --- ray-operator/config/samples/ray_v1alpha1_rayservice.yaml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/ray-operator/config/samples/ray_v1alpha1_rayservice.yaml b/ray-operator/config/samples/ray_v1alpha1_rayservice.yaml index 60db9302a0..d3da3e0211 100644 --- a/ray-operator/config/samples/ray_v1alpha1_rayservice.yaml +++ b/ray-operator/config/samples/ray_v1alpha1_rayservice.yaml @@ -12,7 +12,7 @@ spec: serveConfig: importPath: fruit.deployment_graph runtimeEnv: | - working_dir: "https://github.com/ray-project/test_dag/archive/c620251044717ace0a4c19d766d43c5099af8a77.zip" + working_dir: "https://github.com/ray-project/test_dag/archive/41d09119cbdf8450599f993f51318e9e27c59098.zip" deployments: - name: MangoStand numReplicas: 1 @@ -82,8 +82,6 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - - name: SERVE_DEPLOYMENT_HANDLE_IS_SYNC - value: "1" resources: limits: cpu: 2 @@ -173,8 +171,6 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - - name: SERVE_DEPLOYMENT_HANDLE_IS_SYNC - value: "1" ports: - containerPort: 80 name: client