feat: update RayCluster .status.reason field with pod creation error

fixes #603 ### example before if Pod can't be created due to ResourceQuota exceeded ``` kubectl get rayclusters dxia-test2 -o yaml ... status: state: failed kubectl describe rayclusters dxia-test2 ... Status: State: failed ``` ### example after if Pod can't be created due to ResourceQuota exceeded ``` kubectl get rayclusters dxia-test2 -o yaml ... status: reason: 'pods "dxia-test2-head-lbvdc" is forbidden: exceeded quota: quota, requested: limits.cpu=15,requests.cpu=15, used: limits.cpu=60,requests.cpu=60, limited: limits.cpu=10,requests.cpu=10' state: failed kubectl describe rayclusters dxia-test2 ... Status: Reason: pods "dxia-test2-head-9mdm5" is forbidden: exceeded quota: quota, requested: limits.cpu=15,requests.cpu=15, used: limits.cpu=60,requests.cpu=60, limited: limits.cpu=10,requests.cpu=10 State: failed ```
ray-project · Nov 1, 2022 · ac82334 · ac82334
1 parent 389ba00
commit ac82334
Show file tree

Hide file tree

Showing 9 changed files with 71 additions and 0 deletions.
diff --git a/helm-chart/kuberay-operator/crds/ray.io_rayclusters.yaml b/helm-chart/kuberay-operator/crds/ray.io_rayclusters.yaml
@@ -11138,6 +11138,9 @@ spec:
                   each node group.
                 format: int32
                 type: integer
+              reason:
+                description: Reason provides more information about current State
+                type: string
               state:
                 description: 'INSERT ADDITIONAL STATUS FIELD - define observed state
                   of cluster Important: Run "make" to regenerat'

diff --git a/helm-chart/kuberay-operator/crds/ray.io_rayjobs.yaml b/helm-chart/kuberay-operator/crds/ray.io_rayjobs.yaml
@@ -11696,6 +11696,9 @@ spec:
                       of each node group.
                     format: int32
                     type: integer
+                  reason:
+                    description: Reason provides more information about current State
+                    type: string
                   state:
                     description: 'INSERT ADDITIONAL STATUS FIELD - define observed
                       state of cluster Important: Run "make" to regenerat'

diff --git a/helm-chart/kuberay-operator/crds/ray.io_rayservices.yaml b/helm-chart/kuberay-operator/crds/ray.io_rayservices.yaml
@@ -11752,6 +11752,10 @@ spec:
                           of each node group.
                         format: int32
                         type: integer
+                      reason:
+                        description: Reason provides more information about current
+                          State
+                        type: string
                       state:
                         description: 'INSERT ADDITIONAL STATUS FIELD - define observed
                           state of cluster Important: Run "make" to regenerat'
@@ -11851,6 +11855,10 @@ spec:
                           of each node group.
                         format: int32
                         type: integer
+                      reason:
+                        description: Reason provides more information about current
+                          State
+                        type: string
                       state:
                         description: 'INSERT ADDITIONAL STATUS FIELD - define observed
                           state of cluster Important: Run "make" to regenerat'

diff --git a/ray-operator/apis/ray/v1alpha1/raycluster_types.go b/ray-operator/apis/ray/v1alpha1/raycluster_types.go
@@ -117,6 +117,8 @@ type RayClusterStatus struct {
 	LastUpdateTime *metav1.Time `json:"lastUpdateTime,omitempty"`
 	// Service Endpoints
 	Endpoints map[string]string `json:"endpoints,omitempty"`
+	// Reason provides more information about current State
+	Reason string `json:"reason,omitempty"`
 }
 
 // RayNodeType  the type of a ray node: head/worker

diff --git a/ray-operator/config/crd/bases/ray.io_rayclusters.yaml b/ray-operator/config/crd/bases/ray.io_rayclusters.yaml
@@ -11138,6 +11138,9 @@ spec:
                   each node group.
                 format: int32
                 type: integer
+              reason:
+                description: Reason provides more information about current State
+                type: string
               state:
                 description: 'INSERT ADDITIONAL STATUS FIELD - define observed state
                   of cluster Important: Run "make" to regenerat'

diff --git a/ray-operator/config/crd/bases/ray.io_rayjobs.yaml b/ray-operator/config/crd/bases/ray.io_rayjobs.yaml
@@ -11696,6 +11696,9 @@ spec:
                       of each node group.
                     format: int32
                     type: integer
+                  reason:
+                    description: Reason provides more information about current State
+                    type: string
                   state:
                     description: 'INSERT ADDITIONAL STATUS FIELD - define observed
                       state of cluster Important: Run "make" to regenerat'

diff --git a/ray-operator/config/crd/bases/ray.io_rayservices.yaml b/ray-operator/config/crd/bases/ray.io_rayservices.yaml
@@ -11752,6 +11752,10 @@ spec:
                           of each node group.
                         format: int32
                         type: integer
+                      reason:
+                        description: Reason provides more information about current
+                          State
+                        type: string
                       state:
                         description: 'INSERT ADDITIONAL STATUS FIELD - define observed
                           state of cluster Important: Run "make" to regenerat'
@@ -11851,6 +11855,10 @@ spec:
                           of each node group.
                         format: int32
                         type: integer
+                      reason:
+                        description: Reason provides more information about current
+                          State
+                        type: string
                       state:
                         description: 'INSERT ADDITIONAL STATUS FIELD - define observed
                           state of cluster Important: Run "make" to regenerat'

diff --git a/ray-operator/controllers/ray/raycluster_controller.go b/ray-operator/controllers/ray/raycluster_controller.go
@@ -214,6 +214,9 @@ func (r *RayClusterReconciler) rayClusterReconcile(request ctrl.Request, instanc
 		if updateErr := r.updateClusterState(instance, rayiov1alpha1.Failed); updateErr != nil {
 			r.Log.Error(updateErr, "RayCluster update state error", "cluster name", request.Name)
 		}
+		if updateErr := r.updateClusterReason(instance, err.Error()); updateErr != nil {
+			r.Log.Error(updateErr, "RayCluster update reason error", "cluster name", request.Name)
+		}
 		return ctrl.Result{RequeueAfter: DefaultRequeueDuration}, err
 	}
 	// update the status if needed
@@ -979,3 +982,8 @@ func (r *RayClusterReconciler) updateClusterState(instance *rayiov1alpha1.RayClu
 	instance.Status.State = clusterState
 	return r.Status().Update(context.Background(), instance)
 }
+
+func (r *RayClusterReconciler) updateClusterReason(instance *rayiov1alpha1.RayCluster, clusterReason string) error {
+	instance.Status.Reason = clusterReason
+	return r.Status().Update(context.Background(), instance)
+}
diff --git a/ray-operator/controllers/ray/raycluster_controller_fake_test.go b/ray-operator/controllers/ray/raycluster_controller_fake_test.go
@@ -753,6 +753,39 @@ func TestReconcile_AutoscalerRoleBinding(t *testing.T) {
 	assert.Nil(t, err, "Fail to get autoscaler RoleBinding after reconciliation")
 }
 
+func TestReconcile_UpdateClusterReason(t *testing.T) {
+	setupTest(t)
+	defer tearDown(t)
+	newScheme := runtime.NewScheme()
+	_ = rayiov1alpha1.AddToScheme(newScheme)
+
+	fakeClient := clientFake.NewClientBuilder().WithScheme(newScheme).WithRuntimeObjects(testRayCluster).Build()
+
+	namespacedName := types.NamespacedName{
+		Name:      instanceName,
+		Namespace: namespaceStr,
+	}
+	cluster := rayiov1alpha1.RayCluster{}
+	err := fakeClient.Get(context.Background(), namespacedName, &cluster)
+	assert.Nil(t, err, "Fail to get RayCluster")
+	assert.Empty(t, cluster.Status.Reason, "Cluster reason should be empty")
+
+	testRayClusterReconciler := &RayClusterReconciler{
+		Client:   fakeClient,
+		Recorder: &record.FakeRecorder{},
+		Scheme:   scheme.Scheme,
+		Log:      ctrl.Log.WithName("controllers").WithName("RayCluster"),
+	}
+	reason := "test reason"
+
+	err = testRayClusterReconciler.updateClusterReason(testRayCluster, reason)
+	assert.Nil(t, err, "Fail to update cluster reason")
+
+	err = fakeClient.Get(context.Background(), namespacedName, &cluster)
+	assert.Nil(t, err, "Fail to get RayCluster after updating reason")
+	assert.Equal(t, cluster.Status.Reason, reason, "Cluster reason should be updated")
+}
+
 func TestUpdateEndpoints(t *testing.T) {
 	setupTest(t)
 	defer tearDown(t)