feat: update RayCluster .status.reason field with pod creation error

fixes #603
ray-project · Oct 17, 2022 · ea32961 · ea32961
1 parent 389ba00
commit ea32961
Show file tree

Hide file tree

Showing 5 changed files with 26 additions and 0 deletions.
diff --git a/ray-operator/apis/ray/v1alpha1/raycluster_types.go b/ray-operator/apis/ray/v1alpha1/raycluster_types.go
@@ -117,6 +117,8 @@ type RayClusterStatus struct {
 	LastUpdateTime *metav1.Time `json:"lastUpdateTime,omitempty"`
 	// Service Endpoints
 	Endpoints map[string]string `json:"endpoints,omitempty"`
+	// Reason provides more information about current State
+	Reason string `json:"reason,omitempty"`
 }
 
 // RayNodeType  the type of a ray node: head/worker

diff --git a/ray-operator/config/crd/bases/ray.io_rayclusters.yaml b/ray-operator/config/crd/bases/ray.io_rayclusters.yaml
@@ -11138,6 +11138,9 @@ spec:
                   each node group.
                 format: int32
                 type: integer
+              reason:
+                description: Reason provides more information about current State
+                type: string
               state:
                 description: 'INSERT ADDITIONAL STATUS FIELD - define observed state
                   of cluster Important: Run "make" to regenerat'

diff --git a/ray-operator/config/crd/bases/ray.io_rayjobs.yaml b/ray-operator/config/crd/bases/ray.io_rayjobs.yaml
@@ -11696,6 +11696,9 @@ spec:
                       of each node group.
                     format: int32
                     type: integer
+                  reason:
+                    description: Reason provides more information about current State
+                    type: string
                   state:
                     description: 'INSERT ADDITIONAL STATUS FIELD - define observed
                       state of cluster Important: Run "make" to regenerat'

diff --git a/ray-operator/config/crd/bases/ray.io_rayservices.yaml b/ray-operator/config/crd/bases/ray.io_rayservices.yaml
@@ -11752,6 +11752,10 @@ spec:
                           of each node group.
                         format: int32
                         type: integer
+                      reason:
+                        description: Reason provides more information about current
+                          State
+                        type: string
                       state:
                         description: 'INSERT ADDITIONAL STATUS FIELD - define observed
                           state of cluster Important: Run "make" to regenerat'
@@ -11851,6 +11855,10 @@ spec:
                           of each node group.
                         format: int32
                         type: integer
+                      reason:
+                        description: Reason provides more information about current
+                          State
+                        type: string
                       state:
                         description: 'INSERT ADDITIONAL STATUS FIELD - define observed
                           state of cluster Important: Run "make" to regenerat'

diff --git a/ray-operator/controllers/ray/raycluster_controller.go b/ray-operator/controllers/ray/raycluster_controller.go
@@ -211,9 +211,14 @@ func (r *RayClusterReconciler) rayClusterReconcile(request ctrl.Request, instanc
 		}
 	}
 	if err := r.reconcilePods(instance); err != nil {
+		r.Log.Info("rayClusterReconcile ", "error for cluster", instance.Name)
 		if updateErr := r.updateClusterState(instance, rayiov1alpha1.Failed); updateErr != nil {
 			r.Log.Error(updateErr, "RayCluster update state error", "cluster name", request.Name)
 		}
+		r.Log.Info("rayClusterReconcile ", "trying to update cluster", instance.Name, "reason", err)
+		if updateErr := r.updateClusterReason(instance, err.Error()); updateErr != nil {
+			r.Log.Error(updateErr, "RayCluster update reason error", "cluster name", request.Name)
+		}
 		return ctrl.Result{RequeueAfter: DefaultRequeueDuration}, err
 	}
 	// update the status if needed
@@ -979,3 +984,8 @@ func (r *RayClusterReconciler) updateClusterState(instance *rayiov1alpha1.RayClu
 	instance.Status.State = clusterState
 	return r.Status().Update(context.Background(), instance)
 }
+
+func (r *RayClusterReconciler) updateClusterReason(instance *rayiov1alpha1.RayCluster, clusterReason string) error {
+	instance.Status.Reason = clusterReason
+	return r.Status().Update(context.Background(), instance)
+}