Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add RayCluster.status.metadata field #1

Closed
wants to merge 1 commit into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions apiserver/pkg/model/converter.go
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,8 @@ func FromCrdToApiCluster(cluster *v1alpha1.RayCluster, events []v1.Event) *api.C
for name, port := range cluster.Status.Endpoints {
pbCluster.ServiceEndpoint[name] = port
}

pbCluster.Metadata = cluster.Status.Metadata
return pbCluster
}

Expand Down
3 changes: 3 additions & 0 deletions proto/cluster.proto
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,9 @@ message Cluster {

// Output. The service endpoint of the cluster
map<string, string> service_endpoint = 11;

// Output. Various metadata about the cluster.
map<string, string> metadata = 12;
}

message ClusterSpec {
Expand Down
398 changes: 208 additions & 190 deletions proto/go_client/cluster.pb.go

Large diffs are not rendered by default.

7 changes: 7 additions & 0 deletions proto/swagger/cluster.swagger.json
Original file line number Diff line number Diff line change
Expand Up @@ -299,6 +299,13 @@
"type": "string"
},
"title": "Output. The service endpoint of the cluster"
},
"metadata": {
"type": "object",
"additionalProperties": {
"type": "string"
},
"description": "Output. Various metadata about the cluster."
}
}
},
Expand Down
2 changes: 2 additions & 0 deletions ray-operator/apis/ray/v1alpha1/raycluster_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,8 @@ type RayClusterStatus struct {
LastUpdateTime *metav1.Time `json:"lastUpdateTime,omitempty"`
// Service Endpoints
Endpoints map[string]string `json:"endpoints,omitempty"`
// Metadata
Metadata map[string]string `json:"metadata,omitempty"`

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Here I think it's not necessary for us to add some new spec for the crd, for the metadata we can just store then as some annotation in rayclusters object.

Copy link
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see. @Jeffwan, are you OK with the annotation approach?

}

// RayNodeType the type of a ray node: head/worker
Expand Down
7 changes: 7 additions & 0 deletions ray-operator/apis/ray/v1alpha1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 5 additions & 0 deletions ray-operator/config/crd/bases/ray.io_rayclusters.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -10992,6 +10992,11 @@ spec:
type: string
description: Service Endpoints
type: object
metadata:
additionalProperties:
type: string
description: Metadata
type: object
lastUpdateTime:
description: LastUpdateTime indicates last update timestamp for this
cluster status.
Expand Down
10 changes: 10 additions & 0 deletions ray-operator/config/crd/bases/ray.io_rayservices.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11600,6 +11600,11 @@ spec:
type: string
description: Service Endpoints
type: object
metadata:
additionalProperties:
type: string
description: Metadata
type: object
lastUpdateTime:
description: LastUpdateTime indicates last update timestamp
for this cluster status.
Expand Down Expand Up @@ -11699,6 +11704,11 @@ spec:
type: string
description: Service Endpoints
type: object
metadata:
additionalProperties:
type: string
description: Metadata
type: object
lastUpdateTime:
description: LastUpdateTime indicates last update timestamp
for this cluster status.
Expand Down
2 changes: 2 additions & 0 deletions ray-operator/controllers/ray/common/constant.go
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,8 @@ const (
// Ray health check related configurations
RayAgentRayletHealthPath = "api/local_raylet_healthz"
RayDashboardGCSHealthPath = "api/gcs_healthz"
// Cluster metadata keys
HeadNodeIpKey = "head_node_ip"
)

type ServiceType string
Expand Down
22 changes: 22 additions & 0 deletions ray-operator/controllers/ray/raycluster_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -778,6 +778,10 @@ func (r *RayClusterReconciler) updateStatus(instance *rayiov1alpha1.RayCluster)
return err
}

if err := r.updateMetadata(instance); err != nil {
return err
}

timeNow := metav1.Now()
instance.Status.LastUpdateTime = &timeNow
if err := r.Status().Update(context.Background(), instance); err != nil {
Expand Down Expand Up @@ -955,3 +959,21 @@ func (r *RayClusterReconciler) updateClusterState(instance *rayiov1alpha1.RayClu
instance.Status.State = clusterState
return r.Status().Update(context.Background(), instance)
}

func (r *RayClusterReconciler) updateMetadata(instance *rayiov1alpha1.RayCluster) error {
runtimePods := corev1.PodList{}
filterLabels := client.MatchingLabels{common.RayClusterLabelKey: instance.Name, common.RayNodeTypeLabelKey: string(rayiov1alpha1.HeadNode)}
if err := r.List(context.TODO(), &runtimePods, client.InNamespace(instance.Namespace), filterLabels); err != nil {
return err
}
if len(runtimePods.Items) < 1 {
r.Log.Info("Unable to find head node pod. Not updating status with head IP", "cluster name", instance.Name, "filter labels", filterLabels)
} else if len(runtimePods.Items) > 1 {
r.Log.Info("Found multiple head node pods. Not updating status with head IP", "cluster name", instance.Name, "filter labels", filterLabels)
}

instance.Status.Metadata = map[string]string{
common.HeadNodeIpKey: runtimePods.Items[0].Status.PodIP,
}
return nil
}
24 changes: 24 additions & 0 deletions ray-operator/controllers/ray/raycluster_controller_fake_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,7 @@ func setupTest(t *testing.T) {
},
Status: corev1.PodStatus{
Phase: corev1.PodRunning,
PodIP: "123.123.123.123",
},
},
&corev1.Pod{
Expand Down Expand Up @@ -779,3 +780,26 @@ func TestUpdateEndpoints(t *testing.T) {
}
assert.Equal(t, expected, testRayCluster.Status.Endpoints, "RayCluster status endpoints not updated")
}

func TestUpdateMetadata(t *testing.T) {
setupTest(t)
defer tearDown(t)

fakeClient := clientFake.NewClientBuilder().WithRuntimeObjects(testPods...).Build()

testRayClusterReconciler := &RayClusterReconciler{
Client: fakeClient,
Recorder: &record.FakeRecorder{},
Scheme: scheme.Scheme,
Log: ctrl.Log.WithName("controllers").WithName("RayCluster"),
}

if err := testRayClusterReconciler.updateMetadata(testRayCluster); err != nil {
t.Errorf("updateMetadata failed: %v", err)
}

expected := map[string]string{
"head_node_ip": "123.123.123.123",
}
assert.Equal(t, expected, testRayCluster.Status.Metadata, "RayCluster status.metadata not updated")
}