From b4d0027f708895b1677819582f473364a23f49d7 Mon Sep 17 00:00:00 2001 From: brucez-anyscale Date: Thu, 19 May 2022 10:57:29 -0700 Subject: [PATCH 01/11] KubeRay: kubebuilder api create RayService Controller and CR --- ray-operator/PROJECT | 9 ++ .../rayservice/v1alpha1/groupversion_info.go | 20 ++++ .../rayservice/v1alpha1/rayservice_types.go | 45 +++++++++ .../v1alpha1/zz_generated.deepcopy.go | 99 +++++++++++++++++++ .../crd/bases/rayservice.io_rayservices.yaml | 55 +++++++++++ ray-operator/config/crd/kustomization.yaml | 1 + ray-operator/config/crd/kustomizeconfig.yaml | 19 ++++ ...cainjection_in_rayservice_rayservices.yaml | 7 ++ .../webhook_in_rayservice_rayservices.yaml | 16 +++ .../rayservice_rayservice_editor_role.yaml | 24 +++++ .../rayservice_rayservice_viewer_role.yaml | 20 ++++ ray-operator/config/rbac/role.yaml | 40 ++++++++ .../rayservice_v1alpha1_rayservice.yaml | 6 ++ .../rayservice/rayservice_controller.go | 61 ++++++++++++ .../controllers/rayservice/suite_test.go | 66 +++++++++++++ ray-operator/go.mod | 4 +- ray-operator/main.go | 7 ++ 17 files changed, 497 insertions(+), 2 deletions(-) create mode 100644 ray-operator/apis/rayservice/v1alpha1/groupversion_info.go create mode 100644 ray-operator/apis/rayservice/v1alpha1/rayservice_types.go create mode 100644 ray-operator/apis/rayservice/v1alpha1/zz_generated.deepcopy.go create mode 100644 ray-operator/config/crd/bases/rayservice.io_rayservices.yaml create mode 100644 ray-operator/config/crd/kustomizeconfig.yaml create mode 100644 ray-operator/config/crd/patches/cainjection_in_rayservice_rayservices.yaml create mode 100644 ray-operator/config/crd/patches/webhook_in_rayservice_rayservices.yaml create mode 100644 ray-operator/config/rbac/rayservice_rayservice_editor_role.yaml create mode 100644 ray-operator/config/rbac/rayservice_rayservice_viewer_role.yaml create mode 100644 ray-operator/config/samples/rayservice_v1alpha1_rayservice.yaml create mode 100644 ray-operator/controllers/rayservice/rayservice_controller.go create mode 100644 ray-operator/controllers/rayservice/suite_test.go diff --git a/ray-operator/PROJECT b/ray-operator/PROJECT index 48f765f5e9..fddd684576 100644 --- a/ray-operator/PROJECT +++ b/ray-operator/PROJECT @@ -14,4 +14,13 @@ resources: kind: RayCluster path: github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1 version: v1alpha1 +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: io + group: rayservice + kind: RayService + path: github.com/ray-project/kuberay/ray-operator/apis/rayservice/v1alpha1 + version: v1alpha1 version: "3" diff --git a/ray-operator/apis/rayservice/v1alpha1/groupversion_info.go b/ray-operator/apis/rayservice/v1alpha1/groupversion_info.go new file mode 100644 index 0000000000..d24b38c928 --- /dev/null +++ b/ray-operator/apis/rayservice/v1alpha1/groupversion_info.go @@ -0,0 +1,20 @@ +// Package v1alpha1 contains API Schema definitions for the rayservice v1alpha1 API group +//+kubebuilder:object:generate=true +//+groupName=rayservice.io +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // GroupVersion is group version used to register these objects + GroupVersion = schema.GroupVersion{Group: "rayservice.io", Version: "v1alpha1"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/ray-operator/apis/rayservice/v1alpha1/rayservice_types.go b/ray-operator/apis/rayservice/v1alpha1/rayservice_types.go new file mode 100644 index 0000000000..587fe5ea4c --- /dev/null +++ b/ray-operator/apis/rayservice/v1alpha1/rayservice_types.go @@ -0,0 +1,45 @@ +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! +// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. + +// RayServiceSpec defines the desired state of RayService +type RayServiceSpec struct { + // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster + // Important: Run "make" to regenerate code after modifying this file +} + +// RayServiceStatus defines the observed state of RayService +type RayServiceStatus struct { + // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster + // Important: Run "make" to regenerate code after modifying this file +} + +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status + +// RayService is the Schema for the rayservices API +type RayService struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec RayServiceSpec `json:"spec,omitempty"` + Status RayServiceStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// RayServiceList contains a list of RayService +type RayServiceList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []RayService `json:"items"` +} + +func init() { + SchemeBuilder.Register(&RayService{}, &RayServiceList{}) +} diff --git a/ray-operator/apis/rayservice/v1alpha1/zz_generated.deepcopy.go b/ray-operator/apis/rayservice/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 0000000000..578becf286 --- /dev/null +++ b/ray-operator/apis/rayservice/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,99 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +// Code generated by controller-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RayService) DeepCopyInto(out *RayService) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RayService. +func (in *RayService) DeepCopy() *RayService { + if in == nil { + return nil + } + out := new(RayService) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *RayService) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RayServiceList) DeepCopyInto(out *RayServiceList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]RayService, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RayServiceList. +func (in *RayServiceList) DeepCopy() *RayServiceList { + if in == nil { + return nil + } + out := new(RayServiceList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *RayServiceList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RayServiceSpec) DeepCopyInto(out *RayServiceSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RayServiceSpec. +func (in *RayServiceSpec) DeepCopy() *RayServiceSpec { + if in == nil { + return nil + } + out := new(RayServiceSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RayServiceStatus) DeepCopyInto(out *RayServiceStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RayServiceStatus. +func (in *RayServiceStatus) DeepCopy() *RayServiceStatus { + if in == nil { + return nil + } + out := new(RayServiceStatus) + in.DeepCopyInto(out) + return out +} diff --git a/ray-operator/config/crd/bases/rayservice.io_rayservices.yaml b/ray-operator/config/crd/bases/rayservice.io_rayservices.yaml new file mode 100644 index 0000000000..45e1d10573 --- /dev/null +++ b/ray-operator/config/crd/bases/rayservice.io_rayservices.yaml @@ -0,0 +1,55 @@ + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.6.0 + creationTimestamp: null + name: rayservices.rayservice.io +spec: + group: rayservice.io + names: + kind: RayService + listKind: RayServiceList + plural: rayservices + singular: rayservice + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: RayService is the Schema for the rayservices API + properties: + apiVersion: + description: APIVersion defines the versioned schema of this representation + of an object. + type: string + kind: + description: Kind is a string value representing the REST resource this + object represents. + type: string + metadata: + type: object + spec: + description: RayServiceSpec defines the desired state of RayService + properties: + foo: + description: Foo is an example field of RayService. Edit rayservice_types.go + to remove/update + type: string + type: object + status: + description: RayServiceStatus defines the observed state of RayService + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/ray-operator/config/crd/kustomization.yaml b/ray-operator/config/crd/kustomization.yaml index 34909b74ed..ee35ac2424 100644 --- a/ray-operator/config/crd/kustomization.yaml +++ b/ray-operator/config/crd/kustomization.yaml @@ -3,4 +3,5 @@ # It should be run by config/default resources: - bases/ray.io_rayclusters.yaml +- bases/rayservice.io_rayservices.yaml # +kubebuilder:scaffold:crdkustomizeresource diff --git a/ray-operator/config/crd/kustomizeconfig.yaml b/ray-operator/config/crd/kustomizeconfig.yaml new file mode 100644 index 0000000000..ec5c150a9d --- /dev/null +++ b/ray-operator/config/crd/kustomizeconfig.yaml @@ -0,0 +1,19 @@ +# This file is for teaching kustomize how to substitute name and namespace reference in CRD +nameReference: +- kind: Service + version: v1 + fieldSpecs: + - kind: CustomResourceDefinition + version: v1 + group: apiextensions.k8s.io + path: spec/conversion/webhook/clientConfig/service/name + +namespace: +- kind: CustomResourceDefinition + version: v1 + group: apiextensions.k8s.io + path: spec/conversion/webhook/clientConfig/service/namespace + create: false + +varReference: +- path: metadata/annotations diff --git a/ray-operator/config/crd/patches/cainjection_in_rayservice_rayservices.yaml b/ray-operator/config/crd/patches/cainjection_in_rayservice_rayservices.yaml new file mode 100644 index 0000000000..c418684bca --- /dev/null +++ b/ray-operator/config/crd/patches/cainjection_in_rayservice_rayservices.yaml @@ -0,0 +1,7 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + name: rayservices.rayservice.io diff --git a/ray-operator/config/crd/patches/webhook_in_rayservice_rayservices.yaml b/ray-operator/config/crd/patches/webhook_in_rayservice_rayservices.yaml new file mode 100644 index 0000000000..042a52f987 --- /dev/null +++ b/ray-operator/config/crd/patches/webhook_in_rayservice_rayservices.yaml @@ -0,0 +1,16 @@ +# The following patch enables a conversion webhook for the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: rayservices.rayservice.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + namespace: system + name: webhook-service + path: /convert + conversionReviewVersions: + - v1 diff --git a/ray-operator/config/rbac/rayservice_rayservice_editor_role.yaml b/ray-operator/config/rbac/rayservice_rayservice_editor_role.yaml new file mode 100644 index 0000000000..737e7dd02f --- /dev/null +++ b/ray-operator/config/rbac/rayservice_rayservice_editor_role.yaml @@ -0,0 +1,24 @@ +# permissions for end users to edit rayservices. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: rayservice-editor-role +rules: +- apiGroups: + - rayservice.io + resources: + - rayservices + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - rayservice.io + resources: + - rayservices/status + verbs: + - get diff --git a/ray-operator/config/rbac/rayservice_rayservice_viewer_role.yaml b/ray-operator/config/rbac/rayservice_rayservice_viewer_role.yaml new file mode 100644 index 0000000000..04638aa812 --- /dev/null +++ b/ray-operator/config/rbac/rayservice_rayservice_viewer_role.yaml @@ -0,0 +1,20 @@ +# permissions for end users to view rayservices. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: rayservice-viewer-role +rules: +- apiGroups: + - rayservice.io + resources: + - rayservices + verbs: + - get + - list + - watch +- apiGroups: + - rayservice.io + resources: + - rayservices/status + verbs: + - get diff --git a/ray-operator/config/rbac/role.yaml b/ray-operator/config/rbac/role.yaml index 68fe6c14ac..e4ce571d17 100644 --- a/ray-operator/config/rbac/role.yaml +++ b/ray-operator/config/rbac/role.yaml @@ -55,6 +55,14 @@ rules: - patch - update - watch +- apiGroups: + - "" + resources: + - services/status + verbs: + - get + - patch + - update - apiGroups: - ray.io resources: @@ -67,6 +75,12 @@ rules: - patch - update - watch +- apiGroups: + - ray.io + resources: + - rayclusters/finalizer + verbs: + - update - apiGroups: - ray.io resources: @@ -75,6 +89,32 @@ rules: - get - patch - update +- apiGroups: + - rayservice.io + resources: + - rayservices + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - rayservice.io + resources: + - rayservices/finalizers + verbs: + - update +- apiGroups: + - rayservice.io + resources: + - rayservices/status + verbs: + - get + - patch + - update - apiGroups: - networking.k8s.io resources: diff --git a/ray-operator/config/samples/rayservice_v1alpha1_rayservice.yaml b/ray-operator/config/samples/rayservice_v1alpha1_rayservice.yaml new file mode 100644 index 0000000000..78b5ac787d --- /dev/null +++ b/ray-operator/config/samples/rayservice_v1alpha1_rayservice.yaml @@ -0,0 +1,6 @@ +apiVersion: rayservice.io/v1alpha1 +kind: RayService +metadata: + name: rayservice-sample +spec: + # TODO(user): Add fields here diff --git a/ray-operator/controllers/rayservice/rayservice_controller.go b/ray-operator/controllers/rayservice/rayservice_controller.go new file mode 100644 index 0000000000..591d39bddb --- /dev/null +++ b/ray-operator/controllers/rayservice/rayservice_controller.go @@ -0,0 +1,61 @@ +package rayservice + +import ( + "context" + "github.com/go-logr/logr" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/manager" + + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + + rayservicev1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/rayservice/v1alpha1" +) + +// RayServiceReconciler reconciles a RayService object +type RayServiceReconciler struct { + client.Client + Scheme *runtime.Scheme + Log logr.Logger + Recorder record.EventRecorder +} + +// NewReconciler returns a new reconcile.Reconciler +func NewReconciler(mgr manager.Manager) *RayServiceReconciler { + return &RayServiceReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Log: ctrl.Log.WithName("controllers").WithName("RayService"), + Recorder: mgr.GetEventRecorderFor("rayservice-controller"), + } +} + +//+kubebuilder:rbac:groups=rayservice.io,resources=rayservices,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=rayservice.io,resources=rayservices/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=rayservice.io,resources=rayservices/finalizers,verbs=update + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// TODO(user): Modify the Reconcile function to compare the state specified by +// the RayService object against the actual cluster state, and then +// perform operations to make the cluster state reflect the state specified by +// the user. +// +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.11.2/pkg/reconcile +func (r *RayServiceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + _ = log.FromContext(ctx) + + // TODO(user): your logic here + + return ctrl.Result{}, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *RayServiceReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&rayservicev1alpha1.RayService{}). + Complete(r) +} diff --git a/ray-operator/controllers/rayservice/suite_test.go b/ray-operator/controllers/rayservice/suite_test.go new file mode 100644 index 0000000000..18d0e65e0c --- /dev/null +++ b/ray-operator/controllers/rayservice/suite_test.go @@ -0,0 +1,66 @@ +package rayservice + +import ( + "path/filepath" + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + "sigs.k8s.io/controller-runtime/pkg/envtest/printer" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + rayservicev1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/rayservice/v1alpha1" + //+kubebuilder:scaffold:imports +) + +// These tests use Ginkgo (BDD-style Go testing framework). Refer to +// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. + +var cfg *rest.Config +var k8sClient client.Client +var testEnv *envtest.Environment + +func TestAPIs(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecsWithDefaultAndCustomReporters(t, + "Controller Suite", + []Reporter{printer.NewlineReporter{}}) +} + +var _ = BeforeSuite(func() { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, + ErrorIfCRDPathMissing: true, + } + + var err error + // cfg is defined in this file globally. + cfg, err = testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + + err = rayservicev1alpha1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + //+kubebuilder:scaffold:scheme + + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient).NotTo(BeNil()) + +}, 60) + +var _ = AfterSuite(func() { + By("tearing down the test environment") + err := testEnv.Stop() + Expect(err).NotTo(HaveOccurred()) +}) diff --git a/ray-operator/go.mod b/ray-operator/go.mod index 33c4a9ecfe..90d1ebbfc5 100644 --- a/ray-operator/go.mod +++ b/ray-operator/go.mod @@ -6,8 +6,10 @@ require ( github.com/go-logr/logr v1.2.0 github.com/onsi/ginkgo v1.16.5 github.com/onsi/gomega v1.17.0 + github.com/prometheus/client_golang v1.11.0 github.com/sirupsen/logrus v1.8.1 github.com/stretchr/testify v1.7.0 + go.uber.org/zap v1.19.1 k8s.io/api v0.23.0 k8s.io/apimachinery v0.23.0 k8s.io/client-go v0.23.0 @@ -46,14 +48,12 @@ require ( github.com/nxadm/tail v1.4.8 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_golang v1.11.0 // indirect github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.28.0 // indirect github.com/prometheus/procfs v0.6.0 // indirect github.com/spf13/pflag v1.0.5 // indirect go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect - go.uber.org/zap v1.19.1 // indirect golang.org/x/mod v0.4.2 // indirect golang.org/x/net v0.0.0-20210825183410-e898025ed96a // indirect golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f // indirect diff --git a/ray-operator/main.go b/ray-operator/main.go index e543ead23b..ffaa097793 100644 --- a/ray-operator/main.go +++ b/ray-operator/main.go @@ -6,6 +6,7 @@ import ( "os" "github.com/ray-project/kuberay/ray-operator/controllers/raycluster" + "github.com/ray-project/kuberay/ray-operator/controllers/rayservice" "go.uber.org/zap/zapcore" "sigs.k8s.io/controller-runtime/pkg/healthz" @@ -17,6 +18,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log/zap" rayiov1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/raycluster/v1alpha1" + rayservicev1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/rayservice/v1alpha1" // +kubebuilder:scaffold:imports ) @@ -31,6 +33,7 @@ var ( func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) utilruntime.Must(rayiov1alpha1.AddToScheme(scheme)) + utilruntime.Must(rayservicev1alpha1.AddToScheme(scheme)) // +kubebuilder:scaffold:scheme } @@ -89,6 +92,10 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "RayCluster") os.Exit(1) } + if err = rayservice.NewReconciler(mgr).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "RayService") + os.Exit(1) + } // +kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { From 52eaf3f6aae0c0b5087aabcc740b880c1be059f8 Mon Sep 17 00:00:00 2001 From: brucez-anyscale Date: Thu, 19 May 2022 10:58:17 -0700 Subject: [PATCH 02/11] Update --- ray-operator/config/crd/bases/rayservice.io_rayservices.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/ray-operator/config/crd/bases/rayservice.io_rayservices.yaml b/ray-operator/config/crd/bases/rayservice.io_rayservices.yaml index 45e1d10573..3000669ec6 100644 --- a/ray-operator/config/crd/bases/rayservice.io_rayservices.yaml +++ b/ray-operator/config/crd/bases/rayservice.io_rayservices.yaml @@ -33,11 +33,6 @@ spec: type: object spec: description: RayServiceSpec defines the desired state of RayService - properties: - foo: - description: Foo is an example field of RayService. Edit rayservice_types.go - to remove/update - type: string type: object status: description: RayServiceStatus defines the observed state of RayService From 44c691789082f8d590bd80b15b0813b2c11af3b0 Mon Sep 17 00:00:00 2001 From: brucez-anyscale Date: Thu, 19 May 2022 11:01:02 -0700 Subject: [PATCH 03/11] update --- ray-operator/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ray-operator/Makefile b/ray-operator/Makefile index e11c0884be..7ff551cb82 100644 --- a/ray-operator/Makefile +++ b/ray-operator/Makefile @@ -40,7 +40,7 @@ help: ## Display this help. ##@ Development manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. - $(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases + $(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=kuberay-operator webhook paths="./..." output:crd:artifacts:config=config/crd/bases generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." From 88b1a5a964090739b245c9a61197abf489af99b5 Mon Sep 17 00:00:00 2001 From: brucez-anyscale Date: Thu, 19 May 2022 11:08:36 -0700 Subject: [PATCH 04/11] Auto generate role --- ray-operator/config/rbac/role.yaml | 46 +++++++++---------- .../rayservice/rayservice_controller.go | 12 +++++ 2 files changed, 33 insertions(+), 25 deletions(-) diff --git a/ray-operator/config/rbac/role.yaml b/ray-operator/config/rbac/role.yaml index e4ce571d17..6b01fb4958 100644 --- a/ray-operator/config/rbac/role.yaml +++ b/ray-operator/config/rbac/role.yaml @@ -6,6 +6,15 @@ metadata: creationTimestamp: null name: kuberay-operator rules: +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - list + - update - apiGroups: - "" resources: @@ -42,11 +51,20 @@ rules: - patch - update - watch +- apiGroups: + - "" + resources: + - serviceaccount + verbs: + - create + - delete + - get + - list + - watch - apiGroups: - "" resources: - services - - serviceaccounts verbs: - create - delete @@ -116,45 +134,23 @@ rules: - patch - update - apiGroups: - - networking.k8s.io - resources: - - ingressclasses - verbs: - - get - - list - - watch -- apiGroups: - - extensions - - networking.k8s.io + - rbac.authorization.k8s.io resources: - - ingresses + - rolebindings verbs: - create - delete - get - list - - patch - - update - watch -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - create - - get - - list - - update - apiGroups: - rbac.authorization.k8s.io resources: - roles - - rolebindings verbs: - create - delete - get - list - - patch - update - watch diff --git a/ray-operator/controllers/rayservice/rayservice_controller.go b/ray-operator/controllers/rayservice/rayservice_controller.go index 591d39bddb..fd34a89441 100644 --- a/ray-operator/controllers/rayservice/rayservice_controller.go +++ b/ray-operator/controllers/rayservice/rayservice_controller.go @@ -35,6 +35,18 @@ func NewReconciler(mgr manager.Manager) *RayServiceReconciler { //+kubebuilder:rbac:groups=rayservice.io,resources=rayservices,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=rayservice.io,resources=rayservices/status,verbs=get;update;patch //+kubebuilder:rbac:groups=rayservice.io,resources=rayservices/finalizers,verbs=update +// +kubebuilder:rbac:groups=ray.io,resources=rayclusters,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=ray.io,resources=rayclusters/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=ray.io,resources=rayclusters/finalizer,verbs=update +// +kubebuilder:rbac:groups=core,resources=events,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=core,resources=pods/status,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=core,resources=services/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=coordination.k8s.io,resources=leases,verbs=get;list;create;update +// +kubebuilder:rbac:groups=core,resources=serviceaccount,verbs=get;list;watch;create;delete +// +kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources=roles,verbs=get;list;watch;create;delete;update +// +kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources=rolebindings,verbs=get;list;watch;create;delete // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. From fcb04269a97861657f31d6556a273cec35f7747f Mon Sep 17 00:00:00 2001 From: brucez-anyscale Date: Thu, 19 May 2022 11:14:57 -0700 Subject: [PATCH 05/11] Update role --- ray-operator/config/rbac/role.yaml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/ray-operator/config/rbac/role.yaml b/ray-operator/config/rbac/role.yaml index 6b01fb4958..e4e6b87c2e 100644 --- a/ray-operator/config/rbac/role.yaml +++ b/ray-operator/config/rbac/role.yaml @@ -133,6 +133,27 @@ rules: - get - patch - update +- apiGroups: + - networking.k8s.io + resources: + - ingressclasses + verbs: + - get + - list + - watch +- apiGroups: + - extensions + - networking.k8s.io + resources: + - ingresses + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - rbac.authorization.k8s.io resources: From 95e46eda08595e2eac433216d13baba9ee2cba18 Mon Sep 17 00:00:00 2001 From: brucez-anyscale Date: Thu, 19 May 2022 11:23:33 -0700 Subject: [PATCH 06/11] go imports --- ray-operator/controllers/rayservice/rayservice_controller.go | 1 + 1 file changed, 1 insertion(+) diff --git a/ray-operator/controllers/rayservice/rayservice_controller.go b/ray-operator/controllers/rayservice/rayservice_controller.go index fd34a89441..d7052e41ff 100644 --- a/ray-operator/controllers/rayservice/rayservice_controller.go +++ b/ray-operator/controllers/rayservice/rayservice_controller.go @@ -2,6 +2,7 @@ package rayservice import ( "context" + "github.com/go-logr/logr" "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/manager" From f60f33bc4e73e4d927b2c8500cf544a94a0f2707 Mon Sep 17 00:00:00 2001 From: brucez-anyscale Date: Tue, 24 May 2022 11:32:22 -0700 Subject: [PATCH 07/11] Revert rayservice change --- apiserver/pkg/manager/resource_manager.go | 2 +- apiserver/pkg/model/converter.go | 2 +- apiserver/pkg/util/cluster.go | 2 +- ray-operator/PROJECT | 9 -- ray-operator/README.md | 16 +-- .../apis/{raycluster => ray}/v1alpha1/doc.go | 0 .../v1alpha1/groupversion_info.go | 0 .../v1alpha1/raycluster_types.go | 0 .../v1alpha1/raycluster_types_test.go | 6 +- .../{raycluster => ray}/v1alpha1/register.go | 0 .../v1alpha1/zz_generated.deepcopy.go | 0 .../rayservice/v1alpha1/groupversion_info.go | 20 ---- .../rayservice/v1alpha1/rayservice_types.go | 45 --------- .../v1alpha1/zz_generated.deepcopy.go | 99 ------------------- .../config/crd/bases/ray.io_rayclusters.yaml | 2 +- .../crd/bases/rayservice.io_rayservices.yaml | 50 ---------- ray-operator/config/crd/kustomization.yaml | 1 - ray-operator/config/crd/kustomizeconfig.yaml | 19 ---- ...cainjection_in_rayservice_rayservices.yaml | 7 -- .../webhook_in_rayservice_rayservices.yaml | 16 --- .../rayservice_rayservice_editor_role.yaml | 24 ----- .../rayservice_rayservice_viewer_role.yaml | 20 ---- .../samples/ray-cluster.autoscaler.debug.yaml | 10 +- .../samples/ray-cluster.autoscaler.yaml | 8 +- .../config/samples/ray-cluster.complete.yaml | 14 +-- .../samples/ray-cluster.getting-started.yaml | 6 +- .../samples/ray-cluster.heterogeneous.yaml | 14 +-- .../config/samples/ray-cluster.ingress.yaml | 2 +- .../config/samples/ray-cluster.mini.yaml | 6 +- .../samples/ray-cluster.without-block.yaml | 6 +- .../rayservice_v1alpha1_rayservice.yaml | 6 -- .../{raycluster => ray}/common/constant.go | 2 +- .../{raycluster => ray}/common/ingress.go | 4 +- .../common/ingress_test.go | 8 +- .../{raycluster => ray}/common/metrics.go | 0 .../{raycluster => ray}/common/pod.go | 4 +- .../{raycluster => ray}/common/pod_test.go | 12 +-- .../{raycluster => ray}/common/rbac.go | 4 +- .../{raycluster => ray}/common/service.go | 4 +- .../common/service_test.go | 6 +- .../raycluster_controller.go | 26 ++--- .../raycluster_controller_fake_test.go | 8 +- .../raycluster_controller_test.go | 26 ++--- .../{raycluster => ray}/suite_test.go | 4 +- .../{raycluster => ray}/utils/util.go | 2 +- .../{raycluster => ray}/utils/util_test.go | 10 +- .../rayservice/rayservice_controller.go | 74 -------------- .../controllers/rayservice/suite_test.go | 66 ------------- ray-operator/main.go | 15 +-- .../clientset/versioned/fake/register.go | 2 +- .../clientset/versioned/scheme/register.go | 2 +- .../v1alpha1/fake/fake_raycluster.go | 2 +- .../typed/raycluster/v1alpha1/raycluster.go | 2 +- .../raycluster/v1alpha1/raycluster_client.go | 2 +- .../informers/externalversions/generic.go | 2 +- .../raycluster/v1alpha1/raycluster.go | 2 +- .../listers/raycluster/v1alpha1/raycluster.go | 4 +- tests/config/raycluster-service.yaml | 2 +- 58 files changed, 122 insertions(+), 585 deletions(-) rename ray-operator/apis/{raycluster => ray}/v1alpha1/doc.go (100%) rename ray-operator/apis/{raycluster => ray}/v1alpha1/groupversion_info.go (100%) rename ray-operator/apis/{raycluster => ray}/v1alpha1/raycluster_types.go (100%) rename ray-operator/apis/{raycluster => ray}/v1alpha1/raycluster_types_test.go (95%) rename ray-operator/apis/{raycluster => ray}/v1alpha1/register.go (100%) rename ray-operator/apis/{raycluster => ray}/v1alpha1/zz_generated.deepcopy.go (100%) delete mode 100644 ray-operator/apis/rayservice/v1alpha1/groupversion_info.go delete mode 100644 ray-operator/apis/rayservice/v1alpha1/rayservice_types.go delete mode 100644 ray-operator/apis/rayservice/v1alpha1/zz_generated.deepcopy.go delete mode 100644 ray-operator/config/crd/bases/rayservice.io_rayservices.yaml delete mode 100644 ray-operator/config/crd/kustomizeconfig.yaml delete mode 100644 ray-operator/config/crd/patches/cainjection_in_rayservice_rayservices.yaml delete mode 100644 ray-operator/config/crd/patches/webhook_in_rayservice_rayservices.yaml delete mode 100644 ray-operator/config/rbac/rayservice_rayservice_editor_role.yaml delete mode 100644 ray-operator/config/rbac/rayservice_rayservice_viewer_role.yaml delete mode 100644 ray-operator/config/samples/rayservice_v1alpha1_rayservice.yaml rename ray-operator/controllers/{raycluster => ray}/common/constant.go (92%) rename ray-operator/controllers/{raycluster => ray}/common/ingress.go (96%) rename ray-operator/controllers/{raycluster => ray}/common/ingress_test.go (95%) rename ray-operator/controllers/{raycluster => ray}/common/metrics.go (100%) rename ray-operator/controllers/{raycluster => ray}/common/pod.go (99%) rename ray-operator/controllers/{raycluster => ray}/common/pod_test.go (96%) rename ray-operator/controllers/{raycluster => ray}/common/rbac.go (92%) rename ray-operator/controllers/{raycluster => ray}/common/service.go (96%) rename ray-operator/controllers/{raycluster => ray}/common/service_test.go (95%) rename ray-operator/controllers/{raycluster => ray}/raycluster_controller.go (97%) rename ray-operator/controllers/{raycluster => ray}/raycluster_controller_fake_test.go (99%) rename ray-operator/controllers/{raycluster => ray}/raycluster_controller_test.go (93%) rename ray-operator/controllers/{raycluster => ray}/suite_test.go (98%) rename ray-operator/controllers/{raycluster => ray}/utils/util.go (99%) rename ray-operator/controllers/{raycluster => ray}/utils/util_test.go (93%) delete mode 100644 ray-operator/controllers/rayservice/rayservice_controller.go delete mode 100644 ray-operator/controllers/rayservice/suite_test.go diff --git a/apiserver/pkg/manager/resource_manager.go b/apiserver/pkg/manager/resource_manager.go index 2111313dde..428370d3e7 100644 --- a/apiserver/pkg/manager/resource_manager.go +++ b/apiserver/pkg/manager/resource_manager.go @@ -8,7 +8,7 @@ import ( "github.com/ray-project/kuberay/apiserver/pkg/util" api "github.com/ray-project/kuberay/proto/go_client" - "github.com/ray-project/kuberay/ray-operator/apis/raycluster/v1alpha1" + "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" rayiov1alpha1 "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned/typed/raycluster/v1alpha1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/apiserver/pkg/model/converter.go b/apiserver/pkg/model/converter.go index 7fe75ee6b9..726dfd5e6b 100644 --- a/apiserver/pkg/model/converter.go +++ b/apiserver/pkg/model/converter.go @@ -6,7 +6,7 @@ import ( "github.com/golang/protobuf/ptypes/timestamp" "github.com/ray-project/kuberay/apiserver/pkg/util" api "github.com/ray-project/kuberay/proto/go_client" - "github.com/ray-project/kuberay/ray-operator/apis/raycluster/v1alpha1" + "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" v1 "k8s.io/api/core/v1" ) diff --git a/apiserver/pkg/util/cluster.go b/apiserver/pkg/util/cluster.go index fe04d94d3d..df0df87f10 100644 --- a/apiserver/pkg/util/cluster.go +++ b/apiserver/pkg/util/cluster.go @@ -6,7 +6,7 @@ import ( "strconv" api "github.com/ray-project/kuberay/proto/go_client" - rayclusterapi "github.com/ray-project/kuberay/ray-operator/apis/raycluster/v1alpha1" + rayclusterapi "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/ray-operator/PROJECT b/ray-operator/PROJECT index fddd684576..48f765f5e9 100644 --- a/ray-operator/PROJECT +++ b/ray-operator/PROJECT @@ -14,13 +14,4 @@ resources: kind: RayCluster path: github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1 version: v1alpha1 -- api: - crdVersion: v1 - namespaced: true - controller: true - domain: io - group: rayservice - kind: RayService - path: github.com/ray-project/kuberay/ray-operator/apis/rayservice/v1alpha1 - version: v1alpha1 version: "3" diff --git a/ray-operator/README.md b/ray-operator/README.md index ebd9d14354..9f44a9ea81 100644 --- a/ray-operator/README.md +++ b/ray-operator/README.md @@ -82,27 +82,27 @@ configmap/ray-code created ```shell # Create a cluster. $ kubectl create -f config/samples/ray-cluster.heterogeneous.yaml -raycluster.ray.io/raycluster-heterogeneous created +ray.ray.io/ray-heterogeneous created # List running clusters. $ kubectl get rayclusters NAME AGE -raycluster-heterogeneous 2m48s +ray-heterogeneous 2m48s # The created cluster should include a head pod, worker pod, and a head service. $ kubectl get pods NAME READY STATUS RESTARTS AGE -raycluster-heterogeneous-head-5r6qr 1/1 Running 0 14m -raycluster-heterogeneous-worker-medium-group-ljzzt 1/1 Running 0 14m -raycluster-heterogeneous-worker-small-group-76qxb 1/1 Running 0 14m -raycluster-heterogeneous-worker-small-group-dcl4d 1/1 Running 0 14m +ray-heterogeneous-head-5r6qr 1/1 Running 0 14m +ray-heterogeneous-worker-medium-group-ljzzt 1/1 Running 0 14m +ray-heterogeneous-worker-small-group-76qxb 1/1 Running 0 14m +ray-heterogeneous-worker-small-group-dcl4d 1/1 Running 0 14m ``` ```shell $ kubectl get services NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE kubernetes ClusterIP 10.152.183.1 443/TCP 35d -raycluster-heterogeneous-my-svc ClusterIP None 80/TCP 15m +ray-heterogeneous-my-svc ClusterIP None 80/TCP 15m ``` The logs of the head pod should show 4 nodes in the Ray cluster @@ -112,7 +112,7 @@ Ray Nodes: {'10.1.73.139', '10.1.73.138', '10.1.73.140', '10.1.73.141'} ```shell # check the logs of the head pod -$ kubectl logs raycluster-heterogeneous-head-5r6qr +$ kubectl logs ray-heterogeneous-head-5r6qr 2020-11-18 09:23:32,069 INFO services.py:1092 -- View the Ray dashboard at http://10.1.73.141:8265 2020-11-18 09:23:31,668 INFO scripts.py:467 -- Local node IP: 10.1.73.141 2020-11-18 09:23:32,093 SUCC scripts.py:497 -- -------------------- diff --git a/ray-operator/apis/raycluster/v1alpha1/doc.go b/ray-operator/apis/ray/v1alpha1/doc.go similarity index 100% rename from ray-operator/apis/raycluster/v1alpha1/doc.go rename to ray-operator/apis/ray/v1alpha1/doc.go diff --git a/ray-operator/apis/raycluster/v1alpha1/groupversion_info.go b/ray-operator/apis/ray/v1alpha1/groupversion_info.go similarity index 100% rename from ray-operator/apis/raycluster/v1alpha1/groupversion_info.go rename to ray-operator/apis/ray/v1alpha1/groupversion_info.go diff --git a/ray-operator/apis/raycluster/v1alpha1/raycluster_types.go b/ray-operator/apis/ray/v1alpha1/raycluster_types.go similarity index 100% rename from ray-operator/apis/raycluster/v1alpha1/raycluster_types.go rename to ray-operator/apis/ray/v1alpha1/raycluster_types.go diff --git a/ray-operator/apis/raycluster/v1alpha1/raycluster_types_test.go b/ray-operator/apis/ray/v1alpha1/raycluster_types_test.go similarity index 95% rename from ray-operator/apis/raycluster/v1alpha1/raycluster_types_test.go rename to ray-operator/apis/ray/v1alpha1/raycluster_types_test.go index 1f5fb50311..a599cb5d48 100644 --- a/ray-operator/apis/raycluster/v1alpha1/raycluster_types_test.go +++ b/ray-operator/apis/ray/v1alpha1/raycluster_types_test.go @@ -11,7 +11,7 @@ import ( var myRayCluster = &RayCluster{ ObjectMeta: metav1.ObjectMeta{ - Name: "raycluster-sample", + Name: "ray-sample", Namespace: "default", }, Spec: RayClusterSpec{ @@ -30,7 +30,7 @@ var myRayCluster = &RayCluster{ ObjectMeta: metav1.ObjectMeta{ Namespace: "default", Labels: map[string]string{ - "rayCluster": "raycluster-sample", + "rayCluster": "ray-sample", "groupName": "headgroup", }, }, @@ -71,7 +71,7 @@ var myRayCluster = &RayCluster{ ObjectMeta: metav1.ObjectMeta{ Namespace: "default", Labels: map[string]string{ - "rayCluster": "raycluster-sample", + "rayCluster": "ray-sample", "groupName": "small-group", }, }, diff --git a/ray-operator/apis/raycluster/v1alpha1/register.go b/ray-operator/apis/ray/v1alpha1/register.go similarity index 100% rename from ray-operator/apis/raycluster/v1alpha1/register.go rename to ray-operator/apis/ray/v1alpha1/register.go diff --git a/ray-operator/apis/raycluster/v1alpha1/zz_generated.deepcopy.go b/ray-operator/apis/ray/v1alpha1/zz_generated.deepcopy.go similarity index 100% rename from ray-operator/apis/raycluster/v1alpha1/zz_generated.deepcopy.go rename to ray-operator/apis/ray/v1alpha1/zz_generated.deepcopy.go diff --git a/ray-operator/apis/rayservice/v1alpha1/groupversion_info.go b/ray-operator/apis/rayservice/v1alpha1/groupversion_info.go deleted file mode 100644 index d24b38c928..0000000000 --- a/ray-operator/apis/rayservice/v1alpha1/groupversion_info.go +++ /dev/null @@ -1,20 +0,0 @@ -// Package v1alpha1 contains API Schema definitions for the rayservice v1alpha1 API group -//+kubebuilder:object:generate=true -//+groupName=rayservice.io -package v1alpha1 - -import ( - "k8s.io/apimachinery/pkg/runtime/schema" - "sigs.k8s.io/controller-runtime/pkg/scheme" -) - -var ( - // GroupVersion is group version used to register these objects - GroupVersion = schema.GroupVersion{Group: "rayservice.io", Version: "v1alpha1"} - - // SchemeBuilder is used to add go types to the GroupVersionKind scheme - SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} - - // AddToScheme adds the types in this group-version to the given scheme. - AddToScheme = SchemeBuilder.AddToScheme -) diff --git a/ray-operator/apis/rayservice/v1alpha1/rayservice_types.go b/ray-operator/apis/rayservice/v1alpha1/rayservice_types.go deleted file mode 100644 index 587fe5ea4c..0000000000 --- a/ray-operator/apis/rayservice/v1alpha1/rayservice_types.go +++ /dev/null @@ -1,45 +0,0 @@ -package v1alpha1 - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! -// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. - -// RayServiceSpec defines the desired state of RayService -type RayServiceSpec struct { - // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster - // Important: Run "make" to regenerate code after modifying this file -} - -// RayServiceStatus defines the observed state of RayService -type RayServiceStatus struct { - // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster - // Important: Run "make" to regenerate code after modifying this file -} - -//+kubebuilder:object:root=true -//+kubebuilder:subresource:status - -// RayService is the Schema for the rayservices API -type RayService struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - - Spec RayServiceSpec `json:"spec,omitempty"` - Status RayServiceStatus `json:"status,omitempty"` -} - -//+kubebuilder:object:root=true - -// RayServiceList contains a list of RayService -type RayServiceList struct { - metav1.TypeMeta `json:",inline"` - metav1.ListMeta `json:"metadata,omitempty"` - Items []RayService `json:"items"` -} - -func init() { - SchemeBuilder.Register(&RayService{}, &RayServiceList{}) -} diff --git a/ray-operator/apis/rayservice/v1alpha1/zz_generated.deepcopy.go b/ray-operator/apis/rayservice/v1alpha1/zz_generated.deepcopy.go deleted file mode 100644 index 578becf286..0000000000 --- a/ray-operator/apis/rayservice/v1alpha1/zz_generated.deepcopy.go +++ /dev/null @@ -1,99 +0,0 @@ -//go:build !ignore_autogenerated -// +build !ignore_autogenerated - -// Code generated by controller-gen. DO NOT EDIT. - -package v1alpha1 - -import ( - runtime "k8s.io/apimachinery/pkg/runtime" -) - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *RayService) DeepCopyInto(out *RayService) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - out.Spec = in.Spec - out.Status = in.Status -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RayService. -func (in *RayService) DeepCopy() *RayService { - if in == nil { - return nil - } - out := new(RayService) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *RayService) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *RayServiceList) DeepCopyInto(out *RayServiceList) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ListMeta.DeepCopyInto(&out.ListMeta) - if in.Items != nil { - in, out := &in.Items, &out.Items - *out = make([]RayService, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RayServiceList. -func (in *RayServiceList) DeepCopy() *RayServiceList { - if in == nil { - return nil - } - out := new(RayServiceList) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *RayServiceList) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *RayServiceSpec) DeepCopyInto(out *RayServiceSpec) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RayServiceSpec. -func (in *RayServiceSpec) DeepCopy() *RayServiceSpec { - if in == nil { - return nil - } - out := new(RayServiceSpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *RayServiceStatus) DeepCopyInto(out *RayServiceStatus) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RayServiceStatus. -func (in *RayServiceStatus) DeepCopy() *RayServiceStatus { - if in == nil { - return nil - } - out := new(RayServiceStatus) - in.DeepCopyInto(out) - return out -} diff --git a/ray-operator/config/crd/bases/ray.io_rayclusters.yaml b/ray-operator/config/crd/bases/ray.io_rayclusters.yaml index 7b9e713991..bd89cd0c7b 100644 --- a/ray-operator/config/crd/bases/ray.io_rayclusters.yaml +++ b/ray-operator/config/crd/bases/ray.io_rayclusters.yaml @@ -13,7 +13,7 @@ spec: kind: RayCluster listKind: RayClusterList plural: rayclusters - singular: raycluster + singular: ray scope: Namespaced versions: - name: v1alpha1 diff --git a/ray-operator/config/crd/bases/rayservice.io_rayservices.yaml b/ray-operator/config/crd/bases/rayservice.io_rayservices.yaml deleted file mode 100644 index 3000669ec6..0000000000 --- a/ray-operator/config/crd/bases/rayservice.io_rayservices.yaml +++ /dev/null @@ -1,50 +0,0 @@ - ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.6.0 - creationTimestamp: null - name: rayservices.rayservice.io -spec: - group: rayservice.io - names: - kind: RayService - listKind: RayServiceList - plural: rayservices - singular: rayservice - scope: Namespaced - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: RayService is the Schema for the rayservices API - properties: - apiVersion: - description: APIVersion defines the versioned schema of this representation - of an object. - type: string - kind: - description: Kind is a string value representing the REST resource this - object represents. - type: string - metadata: - type: object - spec: - description: RayServiceSpec defines the desired state of RayService - type: object - status: - description: RayServiceStatus defines the observed state of RayService - type: object - type: object - served: true - storage: true - subresources: - status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/ray-operator/config/crd/kustomization.yaml b/ray-operator/config/crd/kustomization.yaml index ee35ac2424..34909b74ed 100644 --- a/ray-operator/config/crd/kustomization.yaml +++ b/ray-operator/config/crd/kustomization.yaml @@ -3,5 +3,4 @@ # It should be run by config/default resources: - bases/ray.io_rayclusters.yaml -- bases/rayservice.io_rayservices.yaml # +kubebuilder:scaffold:crdkustomizeresource diff --git a/ray-operator/config/crd/kustomizeconfig.yaml b/ray-operator/config/crd/kustomizeconfig.yaml deleted file mode 100644 index ec5c150a9d..0000000000 --- a/ray-operator/config/crd/kustomizeconfig.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# This file is for teaching kustomize how to substitute name and namespace reference in CRD -nameReference: -- kind: Service - version: v1 - fieldSpecs: - - kind: CustomResourceDefinition - version: v1 - group: apiextensions.k8s.io - path: spec/conversion/webhook/clientConfig/service/name - -namespace: -- kind: CustomResourceDefinition - version: v1 - group: apiextensions.k8s.io - path: spec/conversion/webhook/clientConfig/service/namespace - create: false - -varReference: -- path: metadata/annotations diff --git a/ray-operator/config/crd/patches/cainjection_in_rayservice_rayservices.yaml b/ray-operator/config/crd/patches/cainjection_in_rayservice_rayservices.yaml deleted file mode 100644 index c418684bca..0000000000 --- a/ray-operator/config/crd/patches/cainjection_in_rayservice_rayservices.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# The following patch adds a directive for certmanager to inject CA into the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) - name: rayservices.rayservice.io diff --git a/ray-operator/config/crd/patches/webhook_in_rayservice_rayservices.yaml b/ray-operator/config/crd/patches/webhook_in_rayservice_rayservices.yaml deleted file mode 100644 index 042a52f987..0000000000 --- a/ray-operator/config/crd/patches/webhook_in_rayservice_rayservices.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# The following patch enables a conversion webhook for the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: rayservices.rayservice.io -spec: - conversion: - strategy: Webhook - webhook: - clientConfig: - service: - namespace: system - name: webhook-service - path: /convert - conversionReviewVersions: - - v1 diff --git a/ray-operator/config/rbac/rayservice_rayservice_editor_role.yaml b/ray-operator/config/rbac/rayservice_rayservice_editor_role.yaml deleted file mode 100644 index 737e7dd02f..0000000000 --- a/ray-operator/config/rbac/rayservice_rayservice_editor_role.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# permissions for end users to edit rayservices. -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: rayservice-editor-role -rules: -- apiGroups: - - rayservice.io - resources: - - rayservices - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - rayservice.io - resources: - - rayservices/status - verbs: - - get diff --git a/ray-operator/config/rbac/rayservice_rayservice_viewer_role.yaml b/ray-operator/config/rbac/rayservice_rayservice_viewer_role.yaml deleted file mode 100644 index 04638aa812..0000000000 --- a/ray-operator/config/rbac/rayservice_rayservice_viewer_role.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# permissions for end users to view rayservices. -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: rayservice-viewer-role -rules: -- apiGroups: - - rayservice.io - resources: - - rayservices - verbs: - - get - - list - - watch -- apiGroups: - - rayservice.io - resources: - - rayservices/status - verbs: - - get diff --git a/ray-operator/config/samples/ray-cluster.autoscaler.debug.yaml b/ray-operator/config/samples/ray-cluster.autoscaler.debug.yaml index d2798750f1..a3818c6f8d 100644 --- a/ray-operator/config/samples/ray-cluster.autoscaler.debug.yaml +++ b/ray-operator/config/samples/ray-cluster.autoscaler.debug.yaml @@ -6,7 +6,7 @@ metadata: labels: controller-tools.k8s.io: "1.0" # An unique identifier for the head node and workers of this cluster. - name: raycluster-complete + name: ray-complete spec: rayVersion: '1.9.2' enableInTreeAutoscaling: false @@ -96,7 +96,7 @@ spec: # This value must match the metadata.name of the RayCluster CR. # The autoscaler uses this env variable to determine which Ray CR to interact with. # TODO: Match with CR name automatically via operator, Helm, and/or Kustomize. - value: raycluster-complete + value: ray-complete command: ["/home/ray/anaconda3/bin/python"] args: - "/home/ray/run_autoscaler_with_retries.py" @@ -133,9 +133,9 @@ spec: # when a pod is confirmed to be deleted, its name will be removed from the list below #scaleStrategy: # workersToDelete: - # - raycluster-complete-worker-small-group-bdtwh - # - raycluster-complete-worker-small-group-hv457 - # - raycluster-complete-worker-small-group-k8tj7 + # - ray-complete-worker-small-group-bdtwh + # - ray-complete-worker-small-group-hv457 + # - ray-complete-worker-small-group-k8tj7 # the following params are used to complete the ray start: ray start --block --node-ip-address= ... rayStartParams: redis-password: '5241590000000000' diff --git a/ray-operator/config/samples/ray-cluster.autoscaler.yaml b/ray-operator/config/samples/ray-cluster.autoscaler.yaml index 599020bd5f..fce7abf00d 100644 --- a/ray-operator/config/samples/ray-cluster.autoscaler.yaml +++ b/ray-operator/config/samples/ray-cluster.autoscaler.yaml @@ -6,7 +6,7 @@ metadata: labels: controller-tools.k8s.io: "1.0" # An unique identifier for the head node and workers of this cluster. - name: raycluster-autoscaler + name: ray-autoscaler spec: rayVersion: 'nightly' enableInTreeAutoscaling: true @@ -99,9 +99,9 @@ spec: # when a pod is confirmed to be deleted, its name will be removed from the list below #scaleStrategy: # workersToDelete: - # - raycluster-complete-worker-small-group-bdtwh - # - raycluster-complete-worker-small-group-hv457 - # - raycluster-complete-worker-small-group-k8tj7 + # - ray-complete-worker-small-group-bdtwh + # - ray-complete-worker-small-group-hv457 + # - ray-complete-worker-small-group-k8tj7 # the following params are used to complete the ray start: ray start --block --node-ip-address= ... rayStartParams: #redis-password: '5241590000000000' diff --git a/ray-operator/config/samples/ray-cluster.complete.yaml b/ray-operator/config/samples/ray-cluster.complete.yaml index 9fe9e1d30a..f651bdac8a 100644 --- a/ray-operator/config/samples/ray-cluster.complete.yaml +++ b/ray-operator/config/samples/ray-cluster.complete.yaml @@ -4,7 +4,7 @@ metadata: labels: controller-tools.k8s.io: "1.0" # An unique identifier for the head node and workers of this cluster. - name: raycluster-complete + name: ray-complete spec: rayVersion: '1.11.0' ######################headGroupSpecs################################# @@ -29,9 +29,9 @@ spec: template: metadata: labels: - # custom labels. NOTE: do not define custom labels start with `raycluster.`, they may be used in controller. + # custom labels. NOTE: do not define custom labels start with `ray.`, they may be used in controller. # Refer to https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ - rayCluster: raycluster-sample # will be injected if missing + rayCluster: ray-sample # will be injected if missing rayNodeType: head # will be injected if missing, must be head or wroker groupName: headgroup # will be injected if missing # annotations for pod @@ -92,9 +92,9 @@ spec: # when a pod is confirmed to be deleted, its name will be removed from the list below #scaleStrategy: # workersToDelete: - # - raycluster-complete-worker-small-group-bdtwh - # - raycluster-complete-worker-small-group-hv457 - # - raycluster-complete-worker-small-group-k8tj7 + # - ray-complete-worker-small-group-bdtwh + # - ray-complete-worker-small-group-hv457 + # - ray-complete-worker-small-group-k8tj7 # the following params are used to complete the ray start: ray start --block --node-ip-address= ... rayStartParams: redis-password: 'LetMeInRay' # Deprecated since Ray 1.11 due to GCS bootstrapping enabled @@ -104,7 +104,7 @@ spec: template: metadata: labels: - rayCluster: raycluster-complete # will be injected if missing + rayCluster: ray-complete # will be injected if missing rayNodeType: worker # will be injected if missing groupName: small-group # will be injected if missing # annotations for pod diff --git a/ray-operator/config/samples/ray-cluster.getting-started.yaml b/ray-operator/config/samples/ray-cluster.getting-started.yaml index 21e5d71892..2189125aec 100644 --- a/ray-operator/config/samples/ray-cluster.getting-started.yaml +++ b/ray-operator/config/samples/ray-cluster.getting-started.yaml @@ -4,7 +4,7 @@ metadata: labels: controller-tools.k8s.io: "1.0" # An unique identifier for the head node and workers of this cluster. - name: raycluster-getting-started + name: ray-getting-started spec: rayVersion: '1.8.0' # should match the Ray version in the image of the containers ######################headGroupSpecs################################# @@ -30,9 +30,9 @@ spec: template: metadata: labels: - # custom labels. NOTE: do not define custom labels start with `raycluster.`, they may be used in controller. + # custom labels. NOTE: do not define custom labels start with `ray.`, they may be used in controller. # Refer to https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ - rayCluster: raycluster-sample # will be injected if missing + rayCluster: ray-sample # will be injected if missing rayNodeType: head # will be injected if missing, must be head or wroker groupName: headgroup # will be injected if missing # annotations for pod diff --git a/ray-operator/config/samples/ray-cluster.heterogeneous.yaml b/ray-operator/config/samples/ray-cluster.heterogeneous.yaml index bd2b8b89a2..71be32c454 100644 --- a/ray-operator/config/samples/ray-cluster.heterogeneous.yaml +++ b/ray-operator/config/samples/ray-cluster.heterogeneous.yaml @@ -4,7 +4,7 @@ metadata: labels: controller-tools.k8s.io: "1.0" # An unique identifier for the head node and workers of this cluster. - name: raycluster-heterogeneous + name: ray-heterogeneous spec: rayVersion: '1.8.0' ######################headGroupSpecs################################# @@ -32,9 +32,9 @@ spec: template: metadata: labels: - # custom labels. NOTE: do not define custom labels start with `raycluster.`, they may be used in controller. + # custom labels. NOTE: do not define custom labels start with `ray.`, they may be used in controller. # Refer to https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ - rayCluster: raycluster-heterogeneous # will be injected if missing + rayCluster: ray-heterogeneous # will be injected if missing rayNodeType: head # will be injected if missing, must be head or wroker groupName: headgroup # will be injected if missing # annotations for pod @@ -92,7 +92,7 @@ spec: template: metadata: labels: - rayCluster: raycluster-heterogeneous # will be injected if missing + rayCluster: ray-heterogeneous # will be injected if missing rayNodeType: worker # will be injected if missing groupName: small-group # will be injected if missing # annotations for pod @@ -156,7 +156,7 @@ spec: # when a pod is confirmed to be deleted, its name will be removed from the list below # scaleStrategy: # workersToDelete: - #- raycluster-heterogeneous-worker-medium-group-7bv5h + #- ray-heterogeneous-worker-medium-group-7bv5h # - worker-4k2ih # the following params are used to complete the ray start: ray start --block --node-ip-address= ... rayStartParams: @@ -167,7 +167,7 @@ spec: template: metadata: labels: - rayCluster: raycluster-heterogeneous # will be injected if missing + rayCluster: ray-heterogeneous # will be injected if missing rayNodeType: worker # will be injected if missing # annotations for pod annotations: @@ -177,7 +177,7 @@ spec: - name: init-myservice image: busybox:1.28 # Change the cluster postfix if you don't have a default setting - command: ['sh', '-c', "until nslookup raycluster-heterogeneous-head-svc.$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace).svc.cluster.local; do echo waiting for myservice; sleep 2; done"] + command: ['sh', '-c', "until nslookup ray-heterogeneous-head-svc.$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace).svc.cluster.local; do echo waiting for myservice; sleep 2; done"] containers: - name: machine-learning # must consist of lower case alphanumeric characters or '-', and must start and end with an alphanumeric character (e.g. 'my-name', or '123-abc' image: rayproject/ray:1.8.0 diff --git a/ray-operator/config/samples/ray-cluster.ingress.yaml b/ray-operator/config/samples/ray-cluster.ingress.yaml index c1cadbd80c..fc9e4d2a9f 100644 --- a/ray-operator/config/samples/ray-cluster.ingress.yaml +++ b/ray-operator/config/samples/ray-cluster.ingress.yaml @@ -3,7 +3,7 @@ kind: RayCluster metadata: annotations: kubernetes.io/ingress.class: nginx - name: raycluster-ingress + name: ray-ingress spec: rayVersion: '1.6.0' # should match the Ray version in the image of the containers headGroupSpec: diff --git a/ray-operator/config/samples/ray-cluster.mini.yaml b/ray-operator/config/samples/ray-cluster.mini.yaml index 34d8e0b441..f8a301885e 100644 --- a/ray-operator/config/samples/ray-cluster.mini.yaml +++ b/ray-operator/config/samples/ray-cluster.mini.yaml @@ -4,7 +4,7 @@ metadata: labels: controller-tools.k8s.io: "1.0" # An unique identifier for the head node and workers of this cluster. - name: raycluster-mini + name: ray-mini spec: rayVersion: '1.8.0' # should match the Ray version in the image of the containers ######################headGroupSpecs################################# @@ -32,9 +32,9 @@ spec: template: metadata: labels: - # custom labels. NOTE: do not define custom labels start with `raycluster.`, they may be used in controller. + # custom labels. NOTE: do not define custom labels start with `ray.`, they may be used in controller. # Refer to https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ - rayCluster: raycluster-sample # will be injected if missing + rayCluster: ray-sample # will be injected if missing rayNodeType: head # will be injected if missing, must be head or wroker groupName: headgroup # will be injected if missing # annotations for pod diff --git a/ray-operator/config/samples/ray-cluster.without-block.yaml b/ray-operator/config/samples/ray-cluster.without-block.yaml index ee57bfea85..1b4634665f 100644 --- a/ray-operator/config/samples/ray-cluster.without-block.yaml +++ b/ray-operator/config/samples/ray-cluster.without-block.yaml @@ -4,7 +4,7 @@ metadata: labels: controller-tools.k8s.io: "1.0" # An unique identifier for the head node and workers of this cluster. - name: raycluster-non-block + name: ray-non-block spec: rayVersion: '1.8.0' # should match the Ray version in the image of the containers ######################headGroupSpecs################################# @@ -31,9 +31,9 @@ spec: template: metadata: labels: - # custom labels. NOTE: do not define custom labels start with `raycluster.`, they may be used in controller. + # custom labels. NOTE: do not define custom labels start with `ray.`, they may be used in controller. # Refer to https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ - rayCluster: raycluster-sample # will be injected if missing + rayCluster: ray-sample # will be injected if missing rayNodeType: head # will be injected if missing, must be head or wroker groupName: headgroup # will be injected if missing # annotations for pod diff --git a/ray-operator/config/samples/rayservice_v1alpha1_rayservice.yaml b/ray-operator/config/samples/rayservice_v1alpha1_rayservice.yaml deleted file mode 100644 index 78b5ac787d..0000000000 --- a/ray-operator/config/samples/rayservice_v1alpha1_rayservice.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: rayservice.io/v1alpha1 -kind: RayService -metadata: - name: rayservice-sample -spec: - # TODO(user): Add fields here diff --git a/ray-operator/controllers/raycluster/common/constant.go b/ray-operator/controllers/ray/common/constant.go similarity index 92% rename from ray-operator/controllers/raycluster/common/constant.go rename to ray-operator/controllers/ray/common/constant.go index ca5c7704e5..c0f80ea92d 100644 --- a/ray-operator/controllers/raycluster/common/constant.go +++ b/ray-operator/controllers/ray/common/constant.go @@ -8,7 +8,7 @@ const ( RayNodeLabelKey = "ray.io/is-ray-node" RayIDLabelKey = "ray.io/identifier" - // Use as separator for pod name, for example, raycluster-small-size-worker-0 + // Use as separator for pod name, for example, ray-small-size-worker-0 DashSymbol = "-" // Use as default port diff --git a/ray-operator/controllers/raycluster/common/ingress.go b/ray-operator/controllers/ray/common/ingress.go similarity index 96% rename from ray-operator/controllers/raycluster/common/ingress.go rename to ray-operator/controllers/ray/common/ingress.go index 073876168a..fdec99025c 100644 --- a/ray-operator/controllers/raycluster/common/ingress.go +++ b/ray-operator/controllers/ray/common/ingress.go @@ -3,9 +3,9 @@ package common import ( "fmt" - "github.com/ray-project/kuberay/ray-operator/controllers/raycluster/utils" + "github.com/ray-project/kuberay/ray-operator/controllers/ray/utils" - rayiov1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/raycluster/v1alpha1" + rayiov1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" "github.com/sirupsen/logrus" networkingv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/ray-operator/controllers/raycluster/common/ingress_test.go b/ray-operator/controllers/ray/common/ingress_test.go similarity index 95% rename from ray-operator/controllers/raycluster/common/ingress_test.go rename to ray-operator/controllers/ray/common/ingress_test.go index 2dfa11e59a..282a9e1bee 100644 --- a/ray-operator/controllers/raycluster/common/ingress_test.go +++ b/ray-operator/controllers/ray/common/ingress_test.go @@ -4,9 +4,9 @@ import ( "reflect" "testing" - "github.com/ray-project/kuberay/ray-operator/controllers/raycluster/utils" + "github.com/ray-project/kuberay/ray-operator/controllers/ray/utils" - rayiov1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/raycluster/v1alpha1" + rayiov1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" "github.com/stretchr/testify/assert" @@ -17,7 +17,7 @@ import ( var instanceWithIngressEnabled = &rayiov1alpha1.RayCluster{ ObjectMeta: metav1.ObjectMeta{ - Name: "raycluster-sample", + Name: "ray-sample", Namespace: "default", Annotations: map[string]string{ IngressClassAnnotationKey: "nginx", @@ -45,7 +45,7 @@ var instanceWithIngressEnabled = &rayiov1alpha1.RayCluster{ var instanceWithIngressEnabledWithoutIngressClass = &rayiov1alpha1.RayCluster{ ObjectMeta: metav1.ObjectMeta{ - Name: "raycluster-sample", + Name: "ray-sample", Namespace: "default", }, Spec: rayiov1alpha1.RayClusterSpec{ diff --git a/ray-operator/controllers/raycluster/common/metrics.go b/ray-operator/controllers/ray/common/metrics.go similarity index 100% rename from ray-operator/controllers/raycluster/common/metrics.go rename to ray-operator/controllers/ray/common/metrics.go diff --git a/ray-operator/controllers/raycluster/common/pod.go b/ray-operator/controllers/ray/common/pod.go similarity index 99% rename from ray-operator/controllers/raycluster/common/pod.go rename to ray-operator/controllers/ray/common/pod.go index 6e33c58955..3505b69f53 100644 --- a/ray-operator/controllers/raycluster/common/pod.go +++ b/ray-operator/controllers/ray/common/pod.go @@ -6,9 +6,9 @@ import ( "strconv" "strings" - "github.com/ray-project/kuberay/ray-operator/controllers/raycluster/utils" + "github.com/ray-project/kuberay/ray-operator/controllers/ray/utils" - rayiov1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/raycluster/v1alpha1" + rayiov1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" "k8s.io/apimachinery/pkg/api/resource" logf "sigs.k8s.io/controller-runtime/pkg/log" diff --git a/ray-operator/controllers/raycluster/common/pod_test.go b/ray-operator/controllers/ray/common/pod_test.go similarity index 96% rename from ray-operator/controllers/raycluster/common/pod_test.go rename to ray-operator/controllers/ray/common/pod_test.go index d8497b106f..4a01c90dbf 100644 --- a/ray-operator/controllers/raycluster/common/pod_test.go +++ b/ray-operator/controllers/ray/common/pod_test.go @@ -7,19 +7,19 @@ import ( "strings" "testing" - "github.com/ray-project/kuberay/ray-operator/controllers/raycluster/utils" + "github.com/ray-project/kuberay/ray-operator/controllers/ray/utils" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" - rayiov1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/raycluster/v1alpha1" + rayiov1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/pointer" ) var instance = rayiov1alpha1.RayCluster{ ObjectMeta: metav1.ObjectMeta{ - Name: "raycluster-sample", + Name: "ray-sample", Namespace: "default", }, Spec: rayiov1alpha1.RayClusterSpec{ @@ -39,7 +39,7 @@ var instance = rayiov1alpha1.RayCluster{ ObjectMeta: metav1.ObjectMeta{ Namespace: "default", Labels: map[string]string{ - "ray.io/cluster": "raycluster-sample", + "ray.io/cluster": "ray-sample", "ray.io/group": "headgroup", }, }, @@ -79,7 +79,7 @@ var instance = rayiov1alpha1.RayCluster{ ObjectMeta: metav1.ObjectMeta{ Namespace: "default", Labels: map[string]string{ - "ray.io/cluster": "raycluster-sample", + "ray.io/cluster": "ray-sample", "ray.io/group": "small-group", }, }, @@ -191,7 +191,7 @@ func TestBuildPod(t *testing.T) { t.Fatalf("Expected `%v` but got `%v`", expectedResult, actualResult) } - expectedCommandArg := splitAndSort("ulimit -n 65536; ray start --block --num-cpus=1 --address=raycluster-sample-head-svc:6379 --port=6379 --redis-password=LetMeInRay --metrics-export-port=8080") + expectedCommandArg := splitAndSort("ulimit -n 65536; ray start --block --num-cpus=1 --address=ray-sample-head-svc:6379 --port=6379 --redis-password=LetMeInRay --metrics-export-port=8080") if !reflect.DeepEqual(expectedCommandArg, splitAndSort(pod.Spec.Containers[0].Args[0])) { t.Fatalf("Expected `%v` but got `%v`", expectedCommandArg, pod.Spec.Containers[0].Args[0]) } diff --git a/ray-operator/controllers/raycluster/common/rbac.go b/ray-operator/controllers/ray/common/rbac.go similarity index 92% rename from ray-operator/controllers/raycluster/common/rbac.go rename to ray-operator/controllers/ray/common/rbac.go index 1b32ede647..d87d078ca0 100644 --- a/ray-operator/controllers/raycluster/common/rbac.go +++ b/ray-operator/controllers/ray/common/rbac.go @@ -1,8 +1,8 @@ package common import ( - "github.com/ray-project/kuberay/ray-operator/apis/raycluster/v1alpha1" - "github.com/ray-project/kuberay/ray-operator/controllers/raycluster/utils" + "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" + "github.com/ray-project/kuberay/ray-operator/controllers/ray/utils" v1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/ray-operator/controllers/raycluster/common/service.go b/ray-operator/controllers/ray/common/service.go similarity index 96% rename from ray-operator/controllers/raycluster/common/service.go rename to ray-operator/controllers/ray/common/service.go index c6757699eb..5fe7be9e85 100644 --- a/ray-operator/controllers/raycluster/common/service.go +++ b/ray-operator/controllers/ray/common/service.go @@ -1,8 +1,8 @@ package common import ( - rayiov1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/raycluster/v1alpha1" - "github.com/ray-project/kuberay/ray-operator/controllers/raycluster/utils" + rayiov1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" + "github.com/ray-project/kuberay/ray-operator/controllers/ray/utils" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) diff --git a/ray-operator/controllers/raycluster/common/service_test.go b/ray-operator/controllers/ray/common/service_test.go similarity index 95% rename from ray-operator/controllers/raycluster/common/service_test.go rename to ray-operator/controllers/ray/common/service_test.go index 4db8403b39..432e505b2d 100644 --- a/ray-operator/controllers/raycluster/common/service_test.go +++ b/ray-operator/controllers/ray/common/service_test.go @@ -4,7 +4,7 @@ import ( "reflect" "testing" - rayiov1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/raycluster/v1alpha1" + rayiov1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" "github.com/stretchr/testify/assert" @@ -15,7 +15,7 @@ import ( var instanceWithWrongSvc = &rayiov1alpha1.RayCluster{ ObjectMeta: metav1.ObjectMeta{ - Name: "raycluster-sample", + Name: "ray-sample", Namespace: "default", }, Spec: rayiov1alpha1.RayClusterSpec{ @@ -34,7 +34,7 @@ var instanceWithWrongSvc = &rayiov1alpha1.RayCluster{ ObjectMeta: metav1.ObjectMeta{ Namespace: "default", Labels: map[string]string{ - "rayCluster": "raycluster-sample", + "rayCluster": "ray-sample", "groupName": "headgroup", }, }, diff --git a/ray-operator/controllers/raycluster/raycluster_controller.go b/ray-operator/controllers/ray/raycluster_controller.go similarity index 97% rename from ray-operator/controllers/raycluster/raycluster_controller.go rename to ray-operator/controllers/ray/raycluster_controller.go index d838bb5a81..6f559f7c6a 100644 --- a/ray-operator/controllers/raycluster/raycluster_controller.go +++ b/ray-operator/controllers/ray/raycluster_controller.go @@ -1,4 +1,4 @@ -package raycluster +package ray import ( "context" @@ -6,13 +6,13 @@ import ( "strings" "time" - "github.com/ray-project/kuberay/ray-operator/controllers/raycluster/common" - "github.com/ray-project/kuberay/ray-operator/controllers/raycluster/utils" + "github.com/ray-project/kuberay/ray-operator/controllers/ray/common" + "github.com/ray-project/kuberay/ray-operator/controllers/ray/utils" rbacv1 "k8s.io/api/rbac/v1" - rayiov1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/raycluster/v1alpha1" - _ "github.com/ray-project/kuberay/ray-operator/controllers/raycluster/common" + rayiov1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" + _ "github.com/ray-project/kuberay/ray-operator/controllers/ray/common" "k8s.io/client-go/tools/record" "github.com/go-logr/logr" @@ -37,7 +37,7 @@ import ( ) var ( - log = logf.Log.WithName("raycluster-controller") + log = logf.Log.WithName("ray-controller") DefaultRequeueDuration = 2 * time.Second PrioritizeWorkersToDelete bool ) @@ -48,7 +48,7 @@ func NewReconciler(mgr manager.Manager) *RayClusterReconciler { Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Log: ctrl.Log.WithName("controllers").WithName("RayCluster"), - Recorder: mgr.GetEventRecorderFor("raycluster-controller"), + Recorder: mgr.GetEventRecorderFor("ray-controller"), } } @@ -79,7 +79,7 @@ type RayClusterReconciler struct { // +kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources=rolebindings,verbs=get;list;watch;create;delete // Reconcile used to bridge the desired state with the current state func (r *RayClusterReconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) { - _ = r.Log.WithValues("raycluster", request.NamespacedName) + _ = r.Log.WithValues("ray", request.NamespacedName) log.Info("reconciling RayCluster", "cluster name", request.Name) // Fetch the RayCluster instance @@ -496,9 +496,9 @@ func (r *RayClusterReconciler) buildHeadPod(instance rayiov1alpha1.RayCluster) c svcName := utils.GenerateServiceName(instance.Name) podConf := common.DefaultHeadPodTemplate(instance, instance.Spec.HeadGroupSpec, podName, svcName) pod := common.BuildPod(podConf, rayiov1alpha1.HeadNode, instance.Spec.HeadGroupSpec.RayStartParams, svcName) - // Set raycluster instance as the owner and controller + // Set ray instance as the owner and controller if err := controllerutil.SetControllerReference(&instance, &pod, r.Scheme); err != nil { - log.Error(err, "Failed to set controller reference for raycluster pod") + log.Error(err, "Failed to set controller reference for ray pod") } return pod @@ -511,9 +511,9 @@ func (r *RayClusterReconciler) buildWorkerPod(instance rayiov1alpha1.RayCluster, svcName := utils.GenerateServiceName(instance.Name) podTemplateSpec := common.DefaultWorkerPodTemplate(instance, worker, podName, svcName) pod := common.BuildPod(podTemplateSpec, rayiov1alpha1.WorkerNode, worker.RayStartParams, svcName) - // Set raycluster instance as the owner and controller + // Set ray instance as the owner and controller if err := controllerutil.SetControllerReference(&instance, &pod, r.Scheme); err != nil { - log.Error(err, "Failed to set controller reference for raycluster pod") + log.Error(err, "Failed to set controller reference for ray pod") } return pod @@ -522,7 +522,7 @@ func (r *RayClusterReconciler) buildWorkerPod(instance rayiov1alpha1.RayCluster, // SetupWithManager builds the reconciler. func (r *RayClusterReconciler) SetupWithManager(mgr ctrl.Manager, reconcileConcurrency int) error { return ctrl.NewControllerManagedBy(mgr). - For(&rayiov1alpha1.RayCluster{}).Named("raycluster-controller"). + For(&rayiov1alpha1.RayCluster{}).Named("ray-controller"). Watches(&source.Kind{Type: &corev1.Pod{}}, &handler.EnqueueRequestForOwner{ IsController: true, OwnerType: &rayiov1alpha1.RayCluster{}, diff --git a/ray-operator/controllers/raycluster/raycluster_controller_fake_test.go b/ray-operator/controllers/ray/raycluster_controller_fake_test.go similarity index 99% rename from ray-operator/controllers/raycluster/raycluster_controller_fake_test.go rename to ray-operator/controllers/ray/raycluster_controller_fake_test.go index 44aef85ba4..66ee7fb29f 100644 --- a/ray-operator/controllers/raycluster/raycluster_controller_fake_test.go +++ b/ray-operator/controllers/ray/raycluster_controller_fake_test.go @@ -13,16 +13,16 @@ See the License for the specific language governing permissions and limitations under the License. */ -package raycluster +package ray import ( "context" "testing" - "github.com/ray-project/kuberay/ray-operator/controllers/raycluster/common" + "github.com/ray-project/kuberay/ray-operator/controllers/ray/common" . "github.com/onsi/ginkgo" - rayiov1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/raycluster/v1alpha1" + rayiov1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned/scheme" "github.com/stretchr/testify/assert" "k8s.io/apimachinery/pkg/api/errors" @@ -65,7 +65,7 @@ func setupTest(t *testing.T) { PrioritizeWorkersToDelete = true namespaceStr = "default" - instanceName = "raycluster-sample" + instanceName = "ray-sample" enableInTreeAutoscaling = true headGroupNameStr = "head-group" headGroupServiceAccount = "head-service-account" diff --git a/ray-operator/controllers/raycluster/raycluster_controller_test.go b/ray-operator/controllers/ray/raycluster_controller_test.go similarity index 93% rename from ray-operator/controllers/raycluster/raycluster_controller_test.go rename to ray-operator/controllers/ray/raycluster_controller_test.go index 743692d56a..67d1c22664 100644 --- a/ray-operator/controllers/raycluster/raycluster_controller_test.go +++ b/ray-operator/controllers/ray/raycluster_controller_test.go @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package raycluster +package ray import ( "context" @@ -21,12 +21,12 @@ import ( "reflect" "time" - "github.com/ray-project/kuberay/ray-operator/controllers/raycluster/common" - "github.com/ray-project/kuberay/ray-operator/controllers/raycluster/utils" + "github.com/ray-project/kuberay/ray-operator/controllers/ray/common" + "github.com/ray-project/kuberay/ray-operator/controllers/ray/utils" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - rayiov1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/raycluster/v1alpha1" + rayiov1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" @@ -50,7 +50,7 @@ var _ = Context("Inside the default namespace", func() { myRayCluster := &rayiov1alpha1.RayCluster{ ObjectMeta: metav1.ObjectMeta{ - Name: "raycluster-sample", + Name: "ray-sample", Namespace: "default", }, Spec: rayiov1alpha1.RayClusterSpec{ @@ -131,13 +131,13 @@ var _ = Context("Inside the default namespace", func() { filterLabels := client.MatchingLabels{common.RayClusterLabelKey: myRayCluster.Name, common.RayNodeGroupLabelKey: "small-group"} - Describe("When creating a raycluster", func() { - It("should create a raycluster object", func() { + Describe("When creating a ray", func() { + It("should create a ray object", func() { err := k8sClient.Create(ctx, myRayCluster) Expect(err).NotTo(HaveOccurred(), "failed to create test RayCluster resource") }) - It("should see a raycluster object", func() { + It("should see a ray object", func() { Eventually( getResourceFunc(ctx, client.ObjectKey{Name: myRayCluster.Name, Namespace: "default"}, myRayCluster), time.Second*3, time.Millisecond*500).Should(BeNil(), "My myRayCluster = %v", myRayCluster.Name) @@ -146,7 +146,7 @@ var _ = Context("Inside the default namespace", func() { It("should create a new head service resource", func() { svc := &corev1.Service{} Eventually( - getResourceFunc(ctx, client.ObjectKey{Name: "raycluster-sample-head-svc", Namespace: "default"}, svc), + getResourceFunc(ctx, client.ObjectKey{Name: "ray-sample-head-svc", Namespace: "default"}, svc), time.Second*15, time.Millisecond*500).Should(BeNil(), "My head service = %v", svc) Expect(svc.Spec.Selector[common.RayIDLabelKey]).Should(Equal(utils.GenerateIdentifier(myRayCluster.Name, rayiov1alpha1.HeadNode))) }) @@ -202,11 +202,11 @@ var _ = Context("Inside the default namespace", func() { time.Second*15, time.Millisecond*500).Should(Equal(3), fmt.Sprintf("workerGroup %v", workerPods.Items)) }) - It("should update a raycluster object deleting a random pod", func() { + It("should update a ray object deleting a random pod", func() { // adding a scale down Eventually( getResourceFunc(ctx, client.ObjectKey{Name: myRayCluster.Name, Namespace: "default"}, myRayCluster), - time.Second*3, time.Millisecond*500).Should(BeNil(), "My raycluster = %v", myRayCluster) + time.Second*3, time.Millisecond*500).Should(BeNil(), "My ray = %v", myRayCluster) rep := new(int32) *rep = 2 myRayCluster.Spec.WorkerGroupSpecs[0].Replicas = rep @@ -227,11 +227,11 @@ var _ = Context("Inside the default namespace", func() { time.Second*15, time.Millisecond*500).Should(Equal(2), fmt.Sprintf("workerGroup %v", workerPods.Items)) }) - It("should update a raycluster object", func() { + It("should update a ray object", func() { // adding a scale strategy Eventually( getResourceFunc(ctx, client.ObjectKey{Name: myRayCluster.Name, Namespace: "default"}, myRayCluster), - time.Second*3, time.Millisecond*500).Should(BeNil(), "My raycluster = %v", myRayCluster) + time.Second*3, time.Millisecond*500).Should(BeNil(), "My ray = %v", myRayCluster) podToDelete1 := workerPods.Items[0] rep := new(int32) diff --git a/ray-operator/controllers/raycluster/suite_test.go b/ray-operator/controllers/ray/suite_test.go similarity index 98% rename from ray-operator/controllers/raycluster/suite_test.go rename to ray-operator/controllers/ray/suite_test.go index 2d343aaafc..b85eabebff 100644 --- a/ray-operator/controllers/raycluster/suite_test.go +++ b/ray-operator/controllers/ray/suite_test.go @@ -13,13 +13,13 @@ See the License for the specific language governing permissions and limitations under the License. */ -package raycluster +package ray import ( "path/filepath" "testing" - rayiov1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/raycluster/v1alpha1" + rayiov1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" "sigs.k8s.io/controller-runtime/pkg/envtest/printer" . "github.com/onsi/ginkgo" diff --git a/ray-operator/controllers/raycluster/utils/util.go b/ray-operator/controllers/ray/utils/util.go similarity index 99% rename from ray-operator/controllers/raycluster/utils/util.go rename to ray-operator/controllers/ray/utils/util.go index 8bb965c9f9..5298f8db68 100644 --- a/ray-operator/controllers/raycluster/utils/util.go +++ b/ray-operator/controllers/ray/utils/util.go @@ -8,7 +8,7 @@ import ( "strings" "unicode" - rayiov1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/raycluster/v1alpha1" + rayiov1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" "github.com/sirupsen/logrus" corev1 "k8s.io/api/core/v1" diff --git a/ray-operator/controllers/raycluster/utils/util_test.go b/ray-operator/controllers/ray/utils/util_test.go similarity index 93% rename from ray-operator/controllers/raycluster/utils/util_test.go rename to ray-operator/controllers/ray/utils/util_test.go index abf669ac4d..1c04362afd 100644 --- a/ray-operator/controllers/raycluster/utils/util_test.go +++ b/ray-operator/controllers/ray/utils/util_test.go @@ -5,7 +5,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - rayiov1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/raycluster/v1alpha1" + rayiov1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" ) @@ -61,7 +61,7 @@ func createSomePod() (pod *corev1.Pod) { Kind: "Pod", }, ObjectMeta: metav1.ObjectMeta{ - Name: "raycluster-sample-small-group-worker-0", + Name: "ray-sample-small-group-worker-0", Namespace: "default", }, } @@ -75,7 +75,7 @@ func TestGetHeadGroupServiceAccountName(t *testing.T) { "Ray cluster with head group service account": { input: &rayiov1alpha1.RayCluster{ ObjectMeta: metav1.ObjectMeta{ - Name: "raycluster-sample", + Name: "ray-sample", Namespace: "default", }, Spec: rayiov1alpha1.RayClusterSpec{ @@ -93,7 +93,7 @@ func TestGetHeadGroupServiceAccountName(t *testing.T) { "Ray cluster without head group service account": { input: &rayiov1alpha1.RayCluster{ ObjectMeta: metav1.ObjectMeta{ - Name: "raycluster-sample", + Name: "ray-sample", Namespace: "default", }, Spec: rayiov1alpha1.RayClusterSpec{ @@ -104,7 +104,7 @@ func TestGetHeadGroupServiceAccountName(t *testing.T) { }, }, }, - want: "raycluster-sample", + want: "ray-sample", }, } diff --git a/ray-operator/controllers/rayservice/rayservice_controller.go b/ray-operator/controllers/rayservice/rayservice_controller.go deleted file mode 100644 index d7052e41ff..0000000000 --- a/ray-operator/controllers/rayservice/rayservice_controller.go +++ /dev/null @@ -1,74 +0,0 @@ -package rayservice - -import ( - "context" - - "github.com/go-logr/logr" - "k8s.io/client-go/tools/record" - "sigs.k8s.io/controller-runtime/pkg/manager" - - "k8s.io/apimachinery/pkg/runtime" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/log" - - rayservicev1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/rayservice/v1alpha1" -) - -// RayServiceReconciler reconciles a RayService object -type RayServiceReconciler struct { - client.Client - Scheme *runtime.Scheme - Log logr.Logger - Recorder record.EventRecorder -} - -// NewReconciler returns a new reconcile.Reconciler -func NewReconciler(mgr manager.Manager) *RayServiceReconciler { - return &RayServiceReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Log: ctrl.Log.WithName("controllers").WithName("RayService"), - Recorder: mgr.GetEventRecorderFor("rayservice-controller"), - } -} - -//+kubebuilder:rbac:groups=rayservice.io,resources=rayservices,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=rayservice.io,resources=rayservices/status,verbs=get;update;patch -//+kubebuilder:rbac:groups=rayservice.io,resources=rayservices/finalizers,verbs=update -// +kubebuilder:rbac:groups=ray.io,resources=rayclusters,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=ray.io,resources=rayclusters/status,verbs=get;update;patch -// +kubebuilder:rbac:groups=ray.io,resources=rayclusters/finalizer,verbs=update -// +kubebuilder:rbac:groups=core,resources=events,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=core,resources=pods/status,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=core,resources=services/status,verbs=get;update;patch -// +kubebuilder:rbac:groups=coordination.k8s.io,resources=leases,verbs=get;list;create;update -// +kubebuilder:rbac:groups=core,resources=serviceaccount,verbs=get;list;watch;create;delete -// +kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources=roles,verbs=get;list;watch;create;delete;update -// +kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources=rolebindings,verbs=get;list;watch;create;delete - -// Reconcile is part of the main kubernetes reconciliation loop which aims to -// move the current state of the cluster closer to the desired state. -// TODO(user): Modify the Reconcile function to compare the state specified by -// the RayService object against the actual cluster state, and then -// perform operations to make the cluster state reflect the state specified by -// the user. -// -// For more details, check Reconcile and its Result here: -// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.11.2/pkg/reconcile -func (r *RayServiceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - _ = log.FromContext(ctx) - - // TODO(user): your logic here - - return ctrl.Result{}, nil -} - -// SetupWithManager sets up the controller with the Manager. -func (r *RayServiceReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). - For(&rayservicev1alpha1.RayService{}). - Complete(r) -} diff --git a/ray-operator/controllers/rayservice/suite_test.go b/ray-operator/controllers/rayservice/suite_test.go deleted file mode 100644 index 18d0e65e0c..0000000000 --- a/ray-operator/controllers/rayservice/suite_test.go +++ /dev/null @@ -1,66 +0,0 @@ -package rayservice - -import ( - "path/filepath" - "testing" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "k8s.io/client-go/kubernetes/scheme" - "k8s.io/client-go/rest" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/envtest" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" - logf "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/log/zap" - - rayservicev1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/rayservice/v1alpha1" - //+kubebuilder:scaffold:imports -) - -// These tests use Ginkgo (BDD-style Go testing framework). Refer to -// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. - -var cfg *rest.Config -var k8sClient client.Client -var testEnv *envtest.Environment - -func TestAPIs(t *testing.T) { - RegisterFailHandler(Fail) - - RunSpecsWithDefaultAndCustomReporters(t, - "Controller Suite", - []Reporter{printer.NewlineReporter{}}) -} - -var _ = BeforeSuite(func() { - logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) - - By("bootstrapping test environment") - testEnv = &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, - ErrorIfCRDPathMissing: true, - } - - var err error - // cfg is defined in this file globally. - cfg, err = testEnv.Start() - Expect(err).NotTo(HaveOccurred()) - Expect(cfg).NotTo(BeNil()) - - err = rayservicev1alpha1.AddToScheme(scheme.Scheme) - Expect(err).NotTo(HaveOccurred()) - - //+kubebuilder:scaffold:scheme - - k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) - Expect(err).NotTo(HaveOccurred()) - Expect(k8sClient).NotTo(BeNil()) - -}, 60) - -var _ = AfterSuite(func() { - By("tearing down the test environment") - err := testEnv.Stop() - Expect(err).NotTo(HaveOccurred()) -}) diff --git a/ray-operator/main.go b/ray-operator/main.go index ffaa097793..8e52240ad2 100644 --- a/ray-operator/main.go +++ b/ray-operator/main.go @@ -5,8 +5,7 @@ import ( "fmt" "os" - "github.com/ray-project/kuberay/ray-operator/controllers/raycluster" - "github.com/ray-project/kuberay/ray-operator/controllers/rayservice" + "github.com/ray-project/kuberay/ray-operator/controllers/ray" "go.uber.org/zap/zapcore" "sigs.k8s.io/controller-runtime/pkg/healthz" @@ -17,8 +16,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/log/zap" - rayiov1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/raycluster/v1alpha1" - rayservicev1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/rayservice/v1alpha1" + rayiov1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" // +kubebuilder:scaffold:imports ) @@ -33,7 +31,6 @@ var ( func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) utilruntime.Must(rayiov1alpha1.AddToScheme(scheme)) - utilruntime.Must(rayservicev1alpha1.AddToScheme(scheme)) // +kubebuilder:scaffold:scheme } @@ -55,7 +52,7 @@ func main() { "watch-namespace", "", "Watch custom resources in the namespace, ignore other namespaces. If empty, all namespaces will be watched.") - flag.BoolVar(&raycluster.PrioritizeWorkersToDelete, "prioritize-workers-to-delete", false, + flag.BoolVar(&ray.PrioritizeWorkersToDelete, "prioritize-workers-to-delete", false, "Temporary feature flag - to be deleted after testing") opts := zap.Options{ Development: true, @@ -88,14 +85,10 @@ func main() { os.Exit(1) } - if err = raycluster.NewReconciler(mgr).SetupWithManager(mgr, reconcileConcurrency); err != nil { + if err = ray.NewReconciler(mgr).SetupWithManager(mgr, reconcileConcurrency); err != nil { setupLog.Error(err, "unable to create controller", "controller", "RayCluster") os.Exit(1) } - if err = rayservice.NewReconciler(mgr).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "RayService") - os.Exit(1) - } // +kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { diff --git a/ray-operator/pkg/client/clientset/versioned/fake/register.go b/ray-operator/pkg/client/clientset/versioned/fake/register.go index 0307c42921..c64765715a 100644 --- a/ray-operator/pkg/client/clientset/versioned/fake/register.go +++ b/ray-operator/pkg/client/clientset/versioned/fake/register.go @@ -3,7 +3,7 @@ package fake import ( - rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/raycluster/v1alpha1" + rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" diff --git a/ray-operator/pkg/client/clientset/versioned/scheme/register.go b/ray-operator/pkg/client/clientset/versioned/scheme/register.go index 1660268762..7ea47d1c69 100644 --- a/ray-operator/pkg/client/clientset/versioned/scheme/register.go +++ b/ray-operator/pkg/client/clientset/versioned/scheme/register.go @@ -3,7 +3,7 @@ package scheme import ( - rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/raycluster/v1alpha1" + rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" diff --git a/ray-operator/pkg/client/clientset/versioned/typed/raycluster/v1alpha1/fake/fake_raycluster.go b/ray-operator/pkg/client/clientset/versioned/typed/raycluster/v1alpha1/fake/fake_raycluster.go index 62ec8db2f3..202993b9bf 100644 --- a/ray-operator/pkg/client/clientset/versioned/typed/raycluster/v1alpha1/fake/fake_raycluster.go +++ b/ray-operator/pkg/client/clientset/versioned/typed/raycluster/v1alpha1/fake/fake_raycluster.go @@ -5,7 +5,7 @@ package fake import ( "context" - v1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/raycluster/v1alpha1" + v1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" labels "k8s.io/apimachinery/pkg/labels" schema "k8s.io/apimachinery/pkg/runtime/schema" diff --git a/ray-operator/pkg/client/clientset/versioned/typed/raycluster/v1alpha1/raycluster.go b/ray-operator/pkg/client/clientset/versioned/typed/raycluster/v1alpha1/raycluster.go index 306ce32962..d83e3f8dc4 100644 --- a/ray-operator/pkg/client/clientset/versioned/typed/raycluster/v1alpha1/raycluster.go +++ b/ray-operator/pkg/client/clientset/versioned/typed/raycluster/v1alpha1/raycluster.go @@ -6,7 +6,7 @@ import ( "context" "time" - v1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/raycluster/v1alpha1" + v1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" scheme "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned/scheme" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" diff --git a/ray-operator/pkg/client/clientset/versioned/typed/raycluster/v1alpha1/raycluster_client.go b/ray-operator/pkg/client/clientset/versioned/typed/raycluster/v1alpha1/raycluster_client.go index 57cab9ce15..242e64955f 100644 --- a/ray-operator/pkg/client/clientset/versioned/typed/raycluster/v1alpha1/raycluster_client.go +++ b/ray-operator/pkg/client/clientset/versioned/typed/raycluster/v1alpha1/raycluster_client.go @@ -3,7 +3,7 @@ package v1alpha1 import ( - v1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/raycluster/v1alpha1" + v1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned/scheme" rest "k8s.io/client-go/rest" ) diff --git a/ray-operator/pkg/client/informers/externalversions/generic.go b/ray-operator/pkg/client/informers/externalversions/generic.go index afc6bce698..1c74871b39 100644 --- a/ray-operator/pkg/client/informers/externalversions/generic.go +++ b/ray-operator/pkg/client/informers/externalversions/generic.go @@ -5,7 +5,7 @@ package externalversions import ( "fmt" - v1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/raycluster/v1alpha1" + v1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" schema "k8s.io/apimachinery/pkg/runtime/schema" cache "k8s.io/client-go/tools/cache" ) diff --git a/ray-operator/pkg/client/informers/externalversions/raycluster/v1alpha1/raycluster.go b/ray-operator/pkg/client/informers/externalversions/raycluster/v1alpha1/raycluster.go index 58d854ffd4..803bcfe1a4 100644 --- a/ray-operator/pkg/client/informers/externalversions/raycluster/v1alpha1/raycluster.go +++ b/ray-operator/pkg/client/informers/externalversions/raycluster/v1alpha1/raycluster.go @@ -6,7 +6,7 @@ import ( "context" time "time" - rayclusterv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/raycluster/v1alpha1" + rayclusterv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" versioned "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned" internalinterfaces "github.com/ray-project/kuberay/ray-operator/pkg/client/informers/externalversions/internalinterfaces" v1alpha1 "github.com/ray-project/kuberay/ray-operator/pkg/client/listers/raycluster/v1alpha1" diff --git a/ray-operator/pkg/client/listers/raycluster/v1alpha1/raycluster.go b/ray-operator/pkg/client/listers/raycluster/v1alpha1/raycluster.go index e4354bf943..8a4aa965b9 100644 --- a/ray-operator/pkg/client/listers/raycluster/v1alpha1/raycluster.go +++ b/ray-operator/pkg/client/listers/raycluster/v1alpha1/raycluster.go @@ -3,7 +3,7 @@ package v1alpha1 import ( - v1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/raycluster/v1alpha1" + v1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/tools/cache" @@ -77,7 +77,7 @@ func (s rayClusterNamespaceLister) Get(name string) (*v1alpha1.RayCluster, error return nil, err } if !exists { - return nil, errors.NewNotFound(v1alpha1.Resource("raycluster"), name) + return nil, errors.NewNotFound(v1alpha1.Resource("ray"), name) } return obj.(*v1alpha1.RayCluster), nil } diff --git a/tests/config/raycluster-service.yaml b/tests/config/raycluster-service.yaml index 89a9cd5a41..fbf7aab8b7 100644 --- a/tests/config/raycluster-service.yaml +++ b/tests/config/raycluster-service.yaml @@ -5,7 +5,7 @@ metadata: spec: type: NodePort selector: - rayCluster: raycluster-compatibility-test + rayCluster: ray-compatibility-test ports: - name: redis port: 6379 From 5ce33a498a72acdeee69bfbadb87266fe10463e7 Mon Sep 17 00:00:00 2001 From: brucez-anyscale Date: Tue, 24 May 2022 11:35:33 -0700 Subject: [PATCH 08/11] init ray group RayService --- ray-operator/PROJECT | 9 ++ .../apis/ray/v1alpha1/rayservice_types.go | 48 ++++++++++ .../ray/v1alpha1/zz_generated.deepcopy.go | 89 +++++++++++++++++++ .../config/crd/bases/ray.io_rayclusters.yaml | 2 +- .../config/crd/bases/ray.io_rayservices.yaml | 55 ++++++++++++ ray-operator/config/crd/kustomization.yaml | 1 + ray-operator/config/crd/kustomizeconfig.yaml | 19 ++++ .../cainjection_in_ray_rayservices.yaml | 7 ++ .../patches/webhook_in_ray_rayservices.yaml | 16 ++++ .../rbac/ray_rayservice_editor_role.yaml | 24 +++++ .../rbac/ray_rayservice_viewer_role.yaml | 20 +++++ ray-operator/config/rbac/role.yaml | 27 +----- .../samples/ray_v1alpha1_rayservice.yaml | 6 ++ .../controllers/ray/raycluster_controller.go | 2 +- .../controllers/ray/rayservice_controller.go | 74 +++++++++++++++ ray-operator/controllers/ray/suite_test.go | 8 +- ray-operator/main.go | 6 ++ 17 files changed, 386 insertions(+), 27 deletions(-) create mode 100644 ray-operator/apis/ray/v1alpha1/rayservice_types.go create mode 100644 ray-operator/config/crd/bases/ray.io_rayservices.yaml create mode 100644 ray-operator/config/crd/kustomizeconfig.yaml create mode 100644 ray-operator/config/crd/patches/cainjection_in_ray_rayservices.yaml create mode 100644 ray-operator/config/crd/patches/webhook_in_ray_rayservices.yaml create mode 100644 ray-operator/config/rbac/ray_rayservice_editor_role.yaml create mode 100644 ray-operator/config/rbac/ray_rayservice_viewer_role.yaml create mode 100644 ray-operator/config/samples/ray_v1alpha1_rayservice.yaml create mode 100644 ray-operator/controllers/ray/rayservice_controller.go diff --git a/ray-operator/PROJECT b/ray-operator/PROJECT index 48f765f5e9..4f7a61b7e9 100644 --- a/ray-operator/PROJECT +++ b/ray-operator/PROJECT @@ -14,4 +14,13 @@ resources: kind: RayCluster path: github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1 version: v1alpha1 +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: io + group: ray + kind: RayService + path: github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1 + version: v1alpha1 version: "3" diff --git a/ray-operator/apis/ray/v1alpha1/rayservice_types.go b/ray-operator/apis/ray/v1alpha1/rayservice_types.go new file mode 100644 index 0000000000..b993daf2f8 --- /dev/null +++ b/ray-operator/apis/ray/v1alpha1/rayservice_types.go @@ -0,0 +1,48 @@ +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! +// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. + +// RayServiceSpec defines the desired state of RayService +type RayServiceSpec struct { + // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster + // Important: Run "make" to regenerate code after modifying this file + + // Foo is an example field of RayService. Edit rayservice_types.go to remove/update + Foo string `json:"foo,omitempty"` +} + +// RayServiceStatus defines the observed state of RayService +type RayServiceStatus struct { + // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster + // Important: Run "make" to regenerate code after modifying this file +} + +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status + +// RayService is the Schema for the rayservices API +type RayService struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec RayServiceSpec `json:"spec,omitempty"` + Status RayServiceStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// RayServiceList contains a list of RayService +type RayServiceList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []RayService `json:"items"` +} + +func init() { + SchemeBuilder.Register(&RayService{}, &RayServiceList{}) +} diff --git a/ray-operator/apis/ray/v1alpha1/zz_generated.deepcopy.go b/ray-operator/apis/ray/v1alpha1/zz_generated.deepcopy.go index 08cee115ec..c7e43122d1 100644 --- a/ray-operator/apis/ray/v1alpha1/zz_generated.deepcopy.go +++ b/ray-operator/apis/ray/v1alpha1/zz_generated.deepcopy.go @@ -145,6 +145,95 @@ func (in *RayClusterStatus) DeepCopy() *RayClusterStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RayService) DeepCopyInto(out *RayService) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RayService. +func (in *RayService) DeepCopy() *RayService { + if in == nil { + return nil + } + out := new(RayService) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *RayService) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RayServiceList) DeepCopyInto(out *RayServiceList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]RayService, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RayServiceList. +func (in *RayServiceList) DeepCopy() *RayServiceList { + if in == nil { + return nil + } + out := new(RayServiceList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *RayServiceList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RayServiceSpec) DeepCopyInto(out *RayServiceSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RayServiceSpec. +func (in *RayServiceSpec) DeepCopy() *RayServiceSpec { + if in == nil { + return nil + } + out := new(RayServiceSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RayServiceStatus) DeepCopyInto(out *RayServiceStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RayServiceStatus. +func (in *RayServiceStatus) DeepCopy() *RayServiceStatus { + if in == nil { + return nil + } + out := new(RayServiceStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ScaleStrategy) DeepCopyInto(out *ScaleStrategy) { *out = *in diff --git a/ray-operator/config/crd/bases/ray.io_rayclusters.yaml b/ray-operator/config/crd/bases/ray.io_rayclusters.yaml index bd89cd0c7b..7b9e713991 100644 --- a/ray-operator/config/crd/bases/ray.io_rayclusters.yaml +++ b/ray-operator/config/crd/bases/ray.io_rayclusters.yaml @@ -13,7 +13,7 @@ spec: kind: RayCluster listKind: RayClusterList plural: rayclusters - singular: ray + singular: raycluster scope: Namespaced versions: - name: v1alpha1 diff --git a/ray-operator/config/crd/bases/ray.io_rayservices.yaml b/ray-operator/config/crd/bases/ray.io_rayservices.yaml new file mode 100644 index 0000000000..ba7ba26f7e --- /dev/null +++ b/ray-operator/config/crd/bases/ray.io_rayservices.yaml @@ -0,0 +1,55 @@ + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.6.0 + creationTimestamp: null + name: rayservices.ray.io +spec: + group: ray.io + names: + kind: RayService + listKind: RayServiceList + plural: rayservices + singular: rayservice + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: RayService is the Schema for the rayservices API + properties: + apiVersion: + description: APIVersion defines the versioned schema of this representation + of an object. + type: string + kind: + description: Kind is a string value representing the REST resource this + object represents. + type: string + metadata: + type: object + spec: + description: RayServiceSpec defines the desired state of RayService + properties: + foo: + description: Foo is an example field of RayService. Edit rayservice_types.go + to remove/update + type: string + type: object + status: + description: RayServiceStatus defines the observed state of RayService + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/ray-operator/config/crd/kustomization.yaml b/ray-operator/config/crd/kustomization.yaml index 34909b74ed..da5d93f466 100644 --- a/ray-operator/config/crd/kustomization.yaml +++ b/ray-operator/config/crd/kustomization.yaml @@ -3,4 +3,5 @@ # It should be run by config/default resources: - bases/ray.io_rayclusters.yaml +- bases/ray.io_rayservices.yaml # +kubebuilder:scaffold:crdkustomizeresource diff --git a/ray-operator/config/crd/kustomizeconfig.yaml b/ray-operator/config/crd/kustomizeconfig.yaml new file mode 100644 index 0000000000..ec5c150a9d --- /dev/null +++ b/ray-operator/config/crd/kustomizeconfig.yaml @@ -0,0 +1,19 @@ +# This file is for teaching kustomize how to substitute name and namespace reference in CRD +nameReference: +- kind: Service + version: v1 + fieldSpecs: + - kind: CustomResourceDefinition + version: v1 + group: apiextensions.k8s.io + path: spec/conversion/webhook/clientConfig/service/name + +namespace: +- kind: CustomResourceDefinition + version: v1 + group: apiextensions.k8s.io + path: spec/conversion/webhook/clientConfig/service/namespace + create: false + +varReference: +- path: metadata/annotations diff --git a/ray-operator/config/crd/patches/cainjection_in_ray_rayservices.yaml b/ray-operator/config/crd/patches/cainjection_in_ray_rayservices.yaml new file mode 100644 index 0000000000..d46ece7891 --- /dev/null +++ b/ray-operator/config/crd/patches/cainjection_in_ray_rayservices.yaml @@ -0,0 +1,7 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + name: rayservices.ray.io diff --git a/ray-operator/config/crd/patches/webhook_in_ray_rayservices.yaml b/ray-operator/config/crd/patches/webhook_in_ray_rayservices.yaml new file mode 100644 index 0000000000..16a33aa850 --- /dev/null +++ b/ray-operator/config/crd/patches/webhook_in_ray_rayservices.yaml @@ -0,0 +1,16 @@ +# The following patch enables a conversion webhook for the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: rayservices.ray.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + namespace: system + name: webhook-service + path: /convert + conversionReviewVersions: + - v1 diff --git a/ray-operator/config/rbac/ray_rayservice_editor_role.yaml b/ray-operator/config/rbac/ray_rayservice_editor_role.yaml new file mode 100644 index 0000000000..f01c6085ac --- /dev/null +++ b/ray-operator/config/rbac/ray_rayservice_editor_role.yaml @@ -0,0 +1,24 @@ +# permissions for end users to edit rayservices. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: rayservice-editor-role +rules: +- apiGroups: + - ray.io + resources: + - rayservices + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - ray.io + resources: + - rayservices/status + verbs: + - get diff --git a/ray-operator/config/rbac/ray_rayservice_viewer_role.yaml b/ray-operator/config/rbac/ray_rayservice_viewer_role.yaml new file mode 100644 index 0000000000..2aecab24d7 --- /dev/null +++ b/ray-operator/config/rbac/ray_rayservice_viewer_role.yaml @@ -0,0 +1,20 @@ +# permissions for end users to view rayservices. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: rayservice-viewer-role +rules: +- apiGroups: + - ray.io + resources: + - rayservices + verbs: + - get + - list + - watch +- apiGroups: + - ray.io + resources: + - rayservices/status + verbs: + - get diff --git a/ray-operator/config/rbac/role.yaml b/ray-operator/config/rbac/role.yaml index e4e6b87c2e..5ce947c609 100644 --- a/ray-operator/config/rbac/role.yaml +++ b/ray-operator/config/rbac/role.yaml @@ -108,7 +108,7 @@ rules: - patch - update - apiGroups: - - rayservice.io + - ray.io resources: - rayservices verbs: @@ -120,40 +120,19 @@ rules: - update - watch - apiGroups: - - rayservice.io + - ray.io resources: - rayservices/finalizers verbs: - update - apiGroups: - - rayservice.io + - ray.io resources: - rayservices/status verbs: - get - patch - update -- apiGroups: - - networking.k8s.io - resources: - - ingressclasses - verbs: - - get - - list - - watch -- apiGroups: - - extensions - - networking.k8s.io - resources: - - ingresses - verbs: - - create - - delete - - get - - list - - patch - - update - - watch - apiGroups: - rbac.authorization.k8s.io resources: diff --git a/ray-operator/config/samples/ray_v1alpha1_rayservice.yaml b/ray-operator/config/samples/ray_v1alpha1_rayservice.yaml new file mode 100644 index 0000000000..df42b6f305 --- /dev/null +++ b/ray-operator/config/samples/ray_v1alpha1_rayservice.yaml @@ -0,0 +1,6 @@ +apiVersion: ray.io/v1alpha1 +kind: RayService +metadata: + name: rayservice-sample +spec: + # TODO(user): Add fields here diff --git a/ray-operator/controllers/ray/raycluster_controller.go b/ray-operator/controllers/ray/raycluster_controller.go index 6f559f7c6a..bcfbe7f643 100644 --- a/ray-operator/controllers/ray/raycluster_controller.go +++ b/ray-operator/controllers/ray/raycluster_controller.go @@ -48,7 +48,7 @@ func NewReconciler(mgr manager.Manager) *RayClusterReconciler { Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Log: ctrl.Log.WithName("controllers").WithName("RayCluster"), - Recorder: mgr.GetEventRecorderFor("ray-controller"), + Recorder: mgr.GetEventRecorderFor("raycluster-controller"), } } diff --git a/ray-operator/controllers/ray/rayservice_controller.go b/ray-operator/controllers/ray/rayservice_controller.go new file mode 100644 index 0000000000..22be3c5ab9 --- /dev/null +++ b/ray-operator/controllers/ray/rayservice_controller.go @@ -0,0 +1,74 @@ +package ray + +import ( + "context" + + "github.com/go-logr/logr" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/manager" + + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + logf "sigs.k8s.io/controller-runtime/pkg/log" + + rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" +) + +// RayServiceReconciler reconciles a RayService object +type RayServiceReconciler struct { + client.Client + Scheme *runtime.Scheme + Log logr.Logger + Recorder record.EventRecorder +} + +// NewRayServiceReconciler returns a new reconcile.Reconciler +func NewRayServiceReconciler(mgr manager.Manager) *RayServiceReconciler { + return &RayServiceReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Log: ctrl.Log.WithName("controllers").WithName("RayService"), + Recorder: mgr.GetEventRecorderFor("rayservice-controller"), + } +} + +//+kubebuilder:rbac:groups=ray.io,resources=rayservices,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=ray.io,resources=rayservices/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=ray.io,resources=rayservices/finalizers,verbs=update +// +kubebuilder:rbac:groups=ray.io,resources=rayclusters,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=ray.io,resources=rayclusters/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=ray.io,resources=rayclusters/finalizer,verbs=update +// +kubebuilder:rbac:groups=core,resources=events,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=core,resources=pods/status,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=core,resources=services/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=coordination.k8s.io,resources=leases,verbs=get;list;create;update +// +kubebuilder:rbac:groups=core,resources=serviceaccount,verbs=get;list;watch;create;delete +// +kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources=roles,verbs=get;list;watch;create;delete;update +// +kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources=rolebindings,verbs=get;list;watch;create;delete + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// TODO(user): Modify the Reconcile function to compare the state specified by +// the RayService object against the actual cluster state, and then +// perform operations to make the cluster state reflect the state specified by +// the user. +// +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.11.2/pkg/reconcile +func (r *RayServiceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + _ = logf.FromContext(ctx) + + // TODO(user): your logic here + + return ctrl.Result{}, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *RayServiceReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&rayv1alpha1.RayService{}). + Complete(r) +} diff --git a/ray-operator/controllers/ray/suite_test.go b/ray-operator/controllers/ray/suite_test.go index b85eabebff..1a96d28abb 100644 --- a/ray-operator/controllers/ray/suite_test.go +++ b/ray-operator/controllers/ray/suite_test.go @@ -19,9 +19,10 @@ import ( "path/filepath" "testing" - rayiov1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" "sigs.k8s.io/controller-runtime/pkg/envtest/printer" + rayiov1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ctrl "sigs.k8s.io/controller-runtime" @@ -32,6 +33,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/envtest" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" + + rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" // +kubebuilder:scaffold:imports ) @@ -69,6 +72,9 @@ var _ = BeforeSuite(func(done Done) { err = rayiov1alpha1.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) + err = rayv1alpha1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + // +kubebuilder:scaffold:scheme k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) diff --git a/ray-operator/main.go b/ray-operator/main.go index 8e52240ad2..8900112f1c 100644 --- a/ray-operator/main.go +++ b/ray-operator/main.go @@ -17,6 +17,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log/zap" rayiov1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" + rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" // +kubebuilder:scaffold:imports ) @@ -31,6 +32,7 @@ var ( func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) utilruntime.Must(rayiov1alpha1.AddToScheme(scheme)) + utilruntime.Must(rayv1alpha1.AddToScheme(scheme)) // +kubebuilder:scaffold:scheme } @@ -89,6 +91,10 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "RayCluster") os.Exit(1) } + if err = ray.NewRayServiceReconciler(mgr).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "RayService") + os.Exit(1) + } // +kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { From 9b0e8d8ed571c2dee6b795d58c63c7123b39284f Mon Sep 17 00:00:00 2001 From: brucez-anyscale Date: Tue, 24 May 2022 11:51:18 -0700 Subject: [PATCH 09/11] Revert naming --- ray-operator/README.md | 16 ++++++++-------- .../apis/ray/v1alpha1/raycluster_types_test.go | 6 +++--- .../samples/ray-cluster.autoscaler.debug.yaml | 10 +++++----- .../config/samples/ray-cluster.autoscaler.yaml | 8 ++++---- .../config/samples/ray-cluster.complete.yaml | 14 +++++++------- .../samples/ray-cluster.getting-started.yaml | 6 +++--- .../samples/ray-cluster.heterogeneous.yaml | 14 +++++++------- .../config/samples/ray-cluster.ingress.yaml | 2 +- .../config/samples/ray-cluster.mini.yaml | 6 +++--- .../samples/ray-cluster.without-block.yaml | 6 +++--- .../controllers/ray/common/ingress_test.go | 4 ++-- .../controllers/ray/common/pod_test.go | 8 ++++---- .../controllers/ray/common/service_test.go | 4 ++-- .../controllers/ray/raycluster_controller.go | 10 +++++----- .../ray/raycluster_controller_fake_test.go | 2 +- .../ray/raycluster_controller_test.go | 18 +++++++++--------- ray-operator/controllers/ray/suite_test.go | 5 ----- .../controllers/ray/utils/util_test.go | 8 ++++---- ray-operator/main.go | 2 -- tests/config/raycluster-service.yaml | 2 +- 20 files changed, 72 insertions(+), 79 deletions(-) diff --git a/ray-operator/README.md b/ray-operator/README.md index 9f44a9ea81..ebd9d14354 100644 --- a/ray-operator/README.md +++ b/ray-operator/README.md @@ -82,27 +82,27 @@ configmap/ray-code created ```shell # Create a cluster. $ kubectl create -f config/samples/ray-cluster.heterogeneous.yaml -ray.ray.io/ray-heterogeneous created +raycluster.ray.io/raycluster-heterogeneous created # List running clusters. $ kubectl get rayclusters NAME AGE -ray-heterogeneous 2m48s +raycluster-heterogeneous 2m48s # The created cluster should include a head pod, worker pod, and a head service. $ kubectl get pods NAME READY STATUS RESTARTS AGE -ray-heterogeneous-head-5r6qr 1/1 Running 0 14m -ray-heterogeneous-worker-medium-group-ljzzt 1/1 Running 0 14m -ray-heterogeneous-worker-small-group-76qxb 1/1 Running 0 14m -ray-heterogeneous-worker-small-group-dcl4d 1/1 Running 0 14m +raycluster-heterogeneous-head-5r6qr 1/1 Running 0 14m +raycluster-heterogeneous-worker-medium-group-ljzzt 1/1 Running 0 14m +raycluster-heterogeneous-worker-small-group-76qxb 1/1 Running 0 14m +raycluster-heterogeneous-worker-small-group-dcl4d 1/1 Running 0 14m ``` ```shell $ kubectl get services NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE kubernetes ClusterIP 10.152.183.1 443/TCP 35d -ray-heterogeneous-my-svc ClusterIP None 80/TCP 15m +raycluster-heterogeneous-my-svc ClusterIP None 80/TCP 15m ``` The logs of the head pod should show 4 nodes in the Ray cluster @@ -112,7 +112,7 @@ Ray Nodes: {'10.1.73.139', '10.1.73.138', '10.1.73.140', '10.1.73.141'} ```shell # check the logs of the head pod -$ kubectl logs ray-heterogeneous-head-5r6qr +$ kubectl logs raycluster-heterogeneous-head-5r6qr 2020-11-18 09:23:32,069 INFO services.py:1092 -- View the Ray dashboard at http://10.1.73.141:8265 2020-11-18 09:23:31,668 INFO scripts.py:467 -- Local node IP: 10.1.73.141 2020-11-18 09:23:32,093 SUCC scripts.py:497 -- -------------------- diff --git a/ray-operator/apis/ray/v1alpha1/raycluster_types_test.go b/ray-operator/apis/ray/v1alpha1/raycluster_types_test.go index a599cb5d48..1f5fb50311 100644 --- a/ray-operator/apis/ray/v1alpha1/raycluster_types_test.go +++ b/ray-operator/apis/ray/v1alpha1/raycluster_types_test.go @@ -11,7 +11,7 @@ import ( var myRayCluster = &RayCluster{ ObjectMeta: metav1.ObjectMeta{ - Name: "ray-sample", + Name: "raycluster-sample", Namespace: "default", }, Spec: RayClusterSpec{ @@ -30,7 +30,7 @@ var myRayCluster = &RayCluster{ ObjectMeta: metav1.ObjectMeta{ Namespace: "default", Labels: map[string]string{ - "rayCluster": "ray-sample", + "rayCluster": "raycluster-sample", "groupName": "headgroup", }, }, @@ -71,7 +71,7 @@ var myRayCluster = &RayCluster{ ObjectMeta: metav1.ObjectMeta{ Namespace: "default", Labels: map[string]string{ - "rayCluster": "ray-sample", + "rayCluster": "raycluster-sample", "groupName": "small-group", }, }, diff --git a/ray-operator/config/samples/ray-cluster.autoscaler.debug.yaml b/ray-operator/config/samples/ray-cluster.autoscaler.debug.yaml index a3818c6f8d..d2798750f1 100644 --- a/ray-operator/config/samples/ray-cluster.autoscaler.debug.yaml +++ b/ray-operator/config/samples/ray-cluster.autoscaler.debug.yaml @@ -6,7 +6,7 @@ metadata: labels: controller-tools.k8s.io: "1.0" # An unique identifier for the head node and workers of this cluster. - name: ray-complete + name: raycluster-complete spec: rayVersion: '1.9.2' enableInTreeAutoscaling: false @@ -96,7 +96,7 @@ spec: # This value must match the metadata.name of the RayCluster CR. # The autoscaler uses this env variable to determine which Ray CR to interact with. # TODO: Match with CR name automatically via operator, Helm, and/or Kustomize. - value: ray-complete + value: raycluster-complete command: ["/home/ray/anaconda3/bin/python"] args: - "/home/ray/run_autoscaler_with_retries.py" @@ -133,9 +133,9 @@ spec: # when a pod is confirmed to be deleted, its name will be removed from the list below #scaleStrategy: # workersToDelete: - # - ray-complete-worker-small-group-bdtwh - # - ray-complete-worker-small-group-hv457 - # - ray-complete-worker-small-group-k8tj7 + # - raycluster-complete-worker-small-group-bdtwh + # - raycluster-complete-worker-small-group-hv457 + # - raycluster-complete-worker-small-group-k8tj7 # the following params are used to complete the ray start: ray start --block --node-ip-address= ... rayStartParams: redis-password: '5241590000000000' diff --git a/ray-operator/config/samples/ray-cluster.autoscaler.yaml b/ray-operator/config/samples/ray-cluster.autoscaler.yaml index fce7abf00d..599020bd5f 100644 --- a/ray-operator/config/samples/ray-cluster.autoscaler.yaml +++ b/ray-operator/config/samples/ray-cluster.autoscaler.yaml @@ -6,7 +6,7 @@ metadata: labels: controller-tools.k8s.io: "1.0" # An unique identifier for the head node and workers of this cluster. - name: ray-autoscaler + name: raycluster-autoscaler spec: rayVersion: 'nightly' enableInTreeAutoscaling: true @@ -99,9 +99,9 @@ spec: # when a pod is confirmed to be deleted, its name will be removed from the list below #scaleStrategy: # workersToDelete: - # - ray-complete-worker-small-group-bdtwh - # - ray-complete-worker-small-group-hv457 - # - ray-complete-worker-small-group-k8tj7 + # - raycluster-complete-worker-small-group-bdtwh + # - raycluster-complete-worker-small-group-hv457 + # - raycluster-complete-worker-small-group-k8tj7 # the following params are used to complete the ray start: ray start --block --node-ip-address= ... rayStartParams: #redis-password: '5241590000000000' diff --git a/ray-operator/config/samples/ray-cluster.complete.yaml b/ray-operator/config/samples/ray-cluster.complete.yaml index f651bdac8a..9fe9e1d30a 100644 --- a/ray-operator/config/samples/ray-cluster.complete.yaml +++ b/ray-operator/config/samples/ray-cluster.complete.yaml @@ -4,7 +4,7 @@ metadata: labels: controller-tools.k8s.io: "1.0" # An unique identifier for the head node and workers of this cluster. - name: ray-complete + name: raycluster-complete spec: rayVersion: '1.11.0' ######################headGroupSpecs################################# @@ -29,9 +29,9 @@ spec: template: metadata: labels: - # custom labels. NOTE: do not define custom labels start with `ray.`, they may be used in controller. + # custom labels. NOTE: do not define custom labels start with `raycluster.`, they may be used in controller. # Refer to https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ - rayCluster: ray-sample # will be injected if missing + rayCluster: raycluster-sample # will be injected if missing rayNodeType: head # will be injected if missing, must be head or wroker groupName: headgroup # will be injected if missing # annotations for pod @@ -92,9 +92,9 @@ spec: # when a pod is confirmed to be deleted, its name will be removed from the list below #scaleStrategy: # workersToDelete: - # - ray-complete-worker-small-group-bdtwh - # - ray-complete-worker-small-group-hv457 - # - ray-complete-worker-small-group-k8tj7 + # - raycluster-complete-worker-small-group-bdtwh + # - raycluster-complete-worker-small-group-hv457 + # - raycluster-complete-worker-small-group-k8tj7 # the following params are used to complete the ray start: ray start --block --node-ip-address= ... rayStartParams: redis-password: 'LetMeInRay' # Deprecated since Ray 1.11 due to GCS bootstrapping enabled @@ -104,7 +104,7 @@ spec: template: metadata: labels: - rayCluster: ray-complete # will be injected if missing + rayCluster: raycluster-complete # will be injected if missing rayNodeType: worker # will be injected if missing groupName: small-group # will be injected if missing # annotations for pod diff --git a/ray-operator/config/samples/ray-cluster.getting-started.yaml b/ray-operator/config/samples/ray-cluster.getting-started.yaml index 2189125aec..21e5d71892 100644 --- a/ray-operator/config/samples/ray-cluster.getting-started.yaml +++ b/ray-operator/config/samples/ray-cluster.getting-started.yaml @@ -4,7 +4,7 @@ metadata: labels: controller-tools.k8s.io: "1.0" # An unique identifier for the head node and workers of this cluster. - name: ray-getting-started + name: raycluster-getting-started spec: rayVersion: '1.8.0' # should match the Ray version in the image of the containers ######################headGroupSpecs################################# @@ -30,9 +30,9 @@ spec: template: metadata: labels: - # custom labels. NOTE: do not define custom labels start with `ray.`, they may be used in controller. + # custom labels. NOTE: do not define custom labels start with `raycluster.`, they may be used in controller. # Refer to https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ - rayCluster: ray-sample # will be injected if missing + rayCluster: raycluster-sample # will be injected if missing rayNodeType: head # will be injected if missing, must be head or wroker groupName: headgroup # will be injected if missing # annotations for pod diff --git a/ray-operator/config/samples/ray-cluster.heterogeneous.yaml b/ray-operator/config/samples/ray-cluster.heterogeneous.yaml index 71be32c454..bd2b8b89a2 100644 --- a/ray-operator/config/samples/ray-cluster.heterogeneous.yaml +++ b/ray-operator/config/samples/ray-cluster.heterogeneous.yaml @@ -4,7 +4,7 @@ metadata: labels: controller-tools.k8s.io: "1.0" # An unique identifier for the head node and workers of this cluster. - name: ray-heterogeneous + name: raycluster-heterogeneous spec: rayVersion: '1.8.0' ######################headGroupSpecs################################# @@ -32,9 +32,9 @@ spec: template: metadata: labels: - # custom labels. NOTE: do not define custom labels start with `ray.`, they may be used in controller. + # custom labels. NOTE: do not define custom labels start with `raycluster.`, they may be used in controller. # Refer to https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ - rayCluster: ray-heterogeneous # will be injected if missing + rayCluster: raycluster-heterogeneous # will be injected if missing rayNodeType: head # will be injected if missing, must be head or wroker groupName: headgroup # will be injected if missing # annotations for pod @@ -92,7 +92,7 @@ spec: template: metadata: labels: - rayCluster: ray-heterogeneous # will be injected if missing + rayCluster: raycluster-heterogeneous # will be injected if missing rayNodeType: worker # will be injected if missing groupName: small-group # will be injected if missing # annotations for pod @@ -156,7 +156,7 @@ spec: # when a pod is confirmed to be deleted, its name will be removed from the list below # scaleStrategy: # workersToDelete: - #- ray-heterogeneous-worker-medium-group-7bv5h + #- raycluster-heterogeneous-worker-medium-group-7bv5h # - worker-4k2ih # the following params are used to complete the ray start: ray start --block --node-ip-address= ... rayStartParams: @@ -167,7 +167,7 @@ spec: template: metadata: labels: - rayCluster: ray-heterogeneous # will be injected if missing + rayCluster: raycluster-heterogeneous # will be injected if missing rayNodeType: worker # will be injected if missing # annotations for pod annotations: @@ -177,7 +177,7 @@ spec: - name: init-myservice image: busybox:1.28 # Change the cluster postfix if you don't have a default setting - command: ['sh', '-c', "until nslookup ray-heterogeneous-head-svc.$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace).svc.cluster.local; do echo waiting for myservice; sleep 2; done"] + command: ['sh', '-c', "until nslookup raycluster-heterogeneous-head-svc.$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace).svc.cluster.local; do echo waiting for myservice; sleep 2; done"] containers: - name: machine-learning # must consist of lower case alphanumeric characters or '-', and must start and end with an alphanumeric character (e.g. 'my-name', or '123-abc' image: rayproject/ray:1.8.0 diff --git a/ray-operator/config/samples/ray-cluster.ingress.yaml b/ray-operator/config/samples/ray-cluster.ingress.yaml index fc9e4d2a9f..c1cadbd80c 100644 --- a/ray-operator/config/samples/ray-cluster.ingress.yaml +++ b/ray-operator/config/samples/ray-cluster.ingress.yaml @@ -3,7 +3,7 @@ kind: RayCluster metadata: annotations: kubernetes.io/ingress.class: nginx - name: ray-ingress + name: raycluster-ingress spec: rayVersion: '1.6.0' # should match the Ray version in the image of the containers headGroupSpec: diff --git a/ray-operator/config/samples/ray-cluster.mini.yaml b/ray-operator/config/samples/ray-cluster.mini.yaml index f8a301885e..34d8e0b441 100644 --- a/ray-operator/config/samples/ray-cluster.mini.yaml +++ b/ray-operator/config/samples/ray-cluster.mini.yaml @@ -4,7 +4,7 @@ metadata: labels: controller-tools.k8s.io: "1.0" # An unique identifier for the head node and workers of this cluster. - name: ray-mini + name: raycluster-mini spec: rayVersion: '1.8.0' # should match the Ray version in the image of the containers ######################headGroupSpecs################################# @@ -32,9 +32,9 @@ spec: template: metadata: labels: - # custom labels. NOTE: do not define custom labels start with `ray.`, they may be used in controller. + # custom labels. NOTE: do not define custom labels start with `raycluster.`, they may be used in controller. # Refer to https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ - rayCluster: ray-sample # will be injected if missing + rayCluster: raycluster-sample # will be injected if missing rayNodeType: head # will be injected if missing, must be head or wroker groupName: headgroup # will be injected if missing # annotations for pod diff --git a/ray-operator/config/samples/ray-cluster.without-block.yaml b/ray-operator/config/samples/ray-cluster.without-block.yaml index 1b4634665f..ee57bfea85 100644 --- a/ray-operator/config/samples/ray-cluster.without-block.yaml +++ b/ray-operator/config/samples/ray-cluster.without-block.yaml @@ -4,7 +4,7 @@ metadata: labels: controller-tools.k8s.io: "1.0" # An unique identifier for the head node and workers of this cluster. - name: ray-non-block + name: raycluster-non-block spec: rayVersion: '1.8.0' # should match the Ray version in the image of the containers ######################headGroupSpecs################################# @@ -31,9 +31,9 @@ spec: template: metadata: labels: - # custom labels. NOTE: do not define custom labels start with `ray.`, they may be used in controller. + # custom labels. NOTE: do not define custom labels start with `raycluster.`, they may be used in controller. # Refer to https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ - rayCluster: ray-sample # will be injected if missing + rayCluster: raycluster-sample # will be injected if missing rayNodeType: head # will be injected if missing, must be head or wroker groupName: headgroup # will be injected if missing # annotations for pod diff --git a/ray-operator/controllers/ray/common/ingress_test.go b/ray-operator/controllers/ray/common/ingress_test.go index 282a9e1bee..3da57df60f 100644 --- a/ray-operator/controllers/ray/common/ingress_test.go +++ b/ray-operator/controllers/ray/common/ingress_test.go @@ -17,7 +17,7 @@ import ( var instanceWithIngressEnabled = &rayiov1alpha1.RayCluster{ ObjectMeta: metav1.ObjectMeta{ - Name: "ray-sample", + Name: "raycluster-sample", Namespace: "default", Annotations: map[string]string{ IngressClassAnnotationKey: "nginx", @@ -45,7 +45,7 @@ var instanceWithIngressEnabled = &rayiov1alpha1.RayCluster{ var instanceWithIngressEnabledWithoutIngressClass = &rayiov1alpha1.RayCluster{ ObjectMeta: metav1.ObjectMeta{ - Name: "ray-sample", + Name: "raycluster-sample", Namespace: "default", }, Spec: rayiov1alpha1.RayClusterSpec{ diff --git a/ray-operator/controllers/ray/common/pod_test.go b/ray-operator/controllers/ray/common/pod_test.go index 4a01c90dbf..3c79b8c700 100644 --- a/ray-operator/controllers/ray/common/pod_test.go +++ b/ray-operator/controllers/ray/common/pod_test.go @@ -19,7 +19,7 @@ import ( var instance = rayiov1alpha1.RayCluster{ ObjectMeta: metav1.ObjectMeta{ - Name: "ray-sample", + Name: "raycluster-sample", Namespace: "default", }, Spec: rayiov1alpha1.RayClusterSpec{ @@ -39,7 +39,7 @@ var instance = rayiov1alpha1.RayCluster{ ObjectMeta: metav1.ObjectMeta{ Namespace: "default", Labels: map[string]string{ - "ray.io/cluster": "ray-sample", + "ray.io/cluster": "raycluster-sample", "ray.io/group": "headgroup", }, }, @@ -79,7 +79,7 @@ var instance = rayiov1alpha1.RayCluster{ ObjectMeta: metav1.ObjectMeta{ Namespace: "default", Labels: map[string]string{ - "ray.io/cluster": "ray-sample", + "ray.io/cluster": "raycluster-sample", "ray.io/group": "small-group", }, }, @@ -191,7 +191,7 @@ func TestBuildPod(t *testing.T) { t.Fatalf("Expected `%v` but got `%v`", expectedResult, actualResult) } - expectedCommandArg := splitAndSort("ulimit -n 65536; ray start --block --num-cpus=1 --address=ray-sample-head-svc:6379 --port=6379 --redis-password=LetMeInRay --metrics-export-port=8080") + expectedCommandArg := splitAndSort("ulimit -n 65536; ray start --block --num-cpus=1 --address=raycluster-sample-head-svc:6379 --port=6379 --redis-password=LetMeInRay --metrics-export-port=8080") if !reflect.DeepEqual(expectedCommandArg, splitAndSort(pod.Spec.Containers[0].Args[0])) { t.Fatalf("Expected `%v` but got `%v`", expectedCommandArg, pod.Spec.Containers[0].Args[0]) } diff --git a/ray-operator/controllers/ray/common/service_test.go b/ray-operator/controllers/ray/common/service_test.go index 432e505b2d..72040567d3 100644 --- a/ray-operator/controllers/ray/common/service_test.go +++ b/ray-operator/controllers/ray/common/service_test.go @@ -15,7 +15,7 @@ import ( var instanceWithWrongSvc = &rayiov1alpha1.RayCluster{ ObjectMeta: metav1.ObjectMeta{ - Name: "ray-sample", + Name: "raycluster-sample", Namespace: "default", }, Spec: rayiov1alpha1.RayClusterSpec{ @@ -34,7 +34,7 @@ var instanceWithWrongSvc = &rayiov1alpha1.RayCluster{ ObjectMeta: metav1.ObjectMeta{ Namespace: "default", Labels: map[string]string{ - "rayCluster": "ray-sample", + "rayCluster": "raycluster-sample", "groupName": "headgroup", }, }, diff --git a/ray-operator/controllers/ray/raycluster_controller.go b/ray-operator/controllers/ray/raycluster_controller.go index bcfbe7f643..b31007b6e3 100644 --- a/ray-operator/controllers/ray/raycluster_controller.go +++ b/ray-operator/controllers/ray/raycluster_controller.go @@ -37,7 +37,7 @@ import ( ) var ( - log = logf.Log.WithName("ray-controller") + log = logf.Log.WithName("raycluster-controller") DefaultRequeueDuration = 2 * time.Second PrioritizeWorkersToDelete bool ) @@ -79,7 +79,7 @@ type RayClusterReconciler struct { // +kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources=rolebindings,verbs=get;list;watch;create;delete // Reconcile used to bridge the desired state with the current state func (r *RayClusterReconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) { - _ = r.Log.WithValues("ray", request.NamespacedName) + _ = r.Log.WithValues("raycluster", request.NamespacedName) log.Info("reconciling RayCluster", "cluster name", request.Name) // Fetch the RayCluster instance @@ -496,9 +496,9 @@ func (r *RayClusterReconciler) buildHeadPod(instance rayiov1alpha1.RayCluster) c svcName := utils.GenerateServiceName(instance.Name) podConf := common.DefaultHeadPodTemplate(instance, instance.Spec.HeadGroupSpec, podName, svcName) pod := common.BuildPod(podConf, rayiov1alpha1.HeadNode, instance.Spec.HeadGroupSpec.RayStartParams, svcName) - // Set ray instance as the owner and controller + // Set raycluster instance as the owner and controller if err := controllerutil.SetControllerReference(&instance, &pod, r.Scheme); err != nil { - log.Error(err, "Failed to set controller reference for ray pod") + log.Error(err, "Failed to set controller reference for raycluster pod") } return pod @@ -522,7 +522,7 @@ func (r *RayClusterReconciler) buildWorkerPod(instance rayiov1alpha1.RayCluster, // SetupWithManager builds the reconciler. func (r *RayClusterReconciler) SetupWithManager(mgr ctrl.Manager, reconcileConcurrency int) error { return ctrl.NewControllerManagedBy(mgr). - For(&rayiov1alpha1.RayCluster{}).Named("ray-controller"). + For(&rayiov1alpha1.RayCluster{}).Named("raycluster-controller"). Watches(&source.Kind{Type: &corev1.Pod{}}, &handler.EnqueueRequestForOwner{ IsController: true, OwnerType: &rayiov1alpha1.RayCluster{}, diff --git a/ray-operator/controllers/ray/raycluster_controller_fake_test.go b/ray-operator/controllers/ray/raycluster_controller_fake_test.go index 66ee7fb29f..926fe6d02a 100644 --- a/ray-operator/controllers/ray/raycluster_controller_fake_test.go +++ b/ray-operator/controllers/ray/raycluster_controller_fake_test.go @@ -65,7 +65,7 @@ func setupTest(t *testing.T) { PrioritizeWorkersToDelete = true namespaceStr = "default" - instanceName = "ray-sample" + instanceName = "raycluster-sample" enableInTreeAutoscaling = true headGroupNameStr = "head-group" headGroupServiceAccount = "head-service-account" diff --git a/ray-operator/controllers/ray/raycluster_controller_test.go b/ray-operator/controllers/ray/raycluster_controller_test.go index 67d1c22664..50bb7d01ff 100644 --- a/ray-operator/controllers/ray/raycluster_controller_test.go +++ b/ray-operator/controllers/ray/raycluster_controller_test.go @@ -50,7 +50,7 @@ var _ = Context("Inside the default namespace", func() { myRayCluster := &rayiov1alpha1.RayCluster{ ObjectMeta: metav1.ObjectMeta{ - Name: "ray-sample", + Name: "raycluster-sample", Namespace: "default", }, Spec: rayiov1alpha1.RayClusterSpec{ @@ -131,13 +131,13 @@ var _ = Context("Inside the default namespace", func() { filterLabels := client.MatchingLabels{common.RayClusterLabelKey: myRayCluster.Name, common.RayNodeGroupLabelKey: "small-group"} - Describe("When creating a ray", func() { - It("should create a ray object", func() { + Describe("When creating a raycluster", func() { + It("should create a raycluster object", func() { err := k8sClient.Create(ctx, myRayCluster) Expect(err).NotTo(HaveOccurred(), "failed to create test RayCluster resource") }) - It("should see a ray object", func() { + It("should see a raycluster object", func() { Eventually( getResourceFunc(ctx, client.ObjectKey{Name: myRayCluster.Name, Namespace: "default"}, myRayCluster), time.Second*3, time.Millisecond*500).Should(BeNil(), "My myRayCluster = %v", myRayCluster.Name) @@ -146,7 +146,7 @@ var _ = Context("Inside the default namespace", func() { It("should create a new head service resource", func() { svc := &corev1.Service{} Eventually( - getResourceFunc(ctx, client.ObjectKey{Name: "ray-sample-head-svc", Namespace: "default"}, svc), + getResourceFunc(ctx, client.ObjectKey{Name: "raycluster-sample-head-svc", Namespace: "default"}, svc), time.Second*15, time.Millisecond*500).Should(BeNil(), "My head service = %v", svc) Expect(svc.Spec.Selector[common.RayIDLabelKey]).Should(Equal(utils.GenerateIdentifier(myRayCluster.Name, rayiov1alpha1.HeadNode))) }) @@ -202,11 +202,11 @@ var _ = Context("Inside the default namespace", func() { time.Second*15, time.Millisecond*500).Should(Equal(3), fmt.Sprintf("workerGroup %v", workerPods.Items)) }) - It("should update a ray object deleting a random pod", func() { + It("should update a raycluster object deleting a random pod", func() { // adding a scale down Eventually( getResourceFunc(ctx, client.ObjectKey{Name: myRayCluster.Name, Namespace: "default"}, myRayCluster), - time.Second*3, time.Millisecond*500).Should(BeNil(), "My ray = %v", myRayCluster) + time.Second*3, time.Millisecond*500).Should(BeNil(), "My raycluster = %v", myRayCluster) rep := new(int32) *rep = 2 myRayCluster.Spec.WorkerGroupSpecs[0].Replicas = rep @@ -227,11 +227,11 @@ var _ = Context("Inside the default namespace", func() { time.Second*15, time.Millisecond*500).Should(Equal(2), fmt.Sprintf("workerGroup %v", workerPods.Items)) }) - It("should update a ray object", func() { + It("should update a raycluster object", func() { // adding a scale strategy Eventually( getResourceFunc(ctx, client.ObjectKey{Name: myRayCluster.Name, Namespace: "default"}, myRayCluster), - time.Second*3, time.Millisecond*500).Should(BeNil(), "My ray = %v", myRayCluster) + time.Second*3, time.Millisecond*500).Should(BeNil(), "My raycluster = %v", myRayCluster) podToDelete1 := workerPods.Items[0] rep := new(int32) diff --git a/ray-operator/controllers/ray/suite_test.go b/ray-operator/controllers/ray/suite_test.go index 1a96d28abb..5d2aaf335d 100644 --- a/ray-operator/controllers/ray/suite_test.go +++ b/ray-operator/controllers/ray/suite_test.go @@ -33,8 +33,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/envtest" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" - - rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" // +kubebuilder:scaffold:imports ) @@ -72,9 +70,6 @@ var _ = BeforeSuite(func(done Done) { err = rayiov1alpha1.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) - err = rayv1alpha1.AddToScheme(scheme.Scheme) - Expect(err).NotTo(HaveOccurred()) - // +kubebuilder:scaffold:scheme k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) diff --git a/ray-operator/controllers/ray/utils/util_test.go b/ray-operator/controllers/ray/utils/util_test.go index 1c04362afd..9d5081d102 100644 --- a/ray-operator/controllers/ray/utils/util_test.go +++ b/ray-operator/controllers/ray/utils/util_test.go @@ -61,7 +61,7 @@ func createSomePod() (pod *corev1.Pod) { Kind: "Pod", }, ObjectMeta: metav1.ObjectMeta{ - Name: "ray-sample-small-group-worker-0", + Name: "raycluster-sample-small-group-worker-0", Namespace: "default", }, } @@ -75,7 +75,7 @@ func TestGetHeadGroupServiceAccountName(t *testing.T) { "Ray cluster with head group service account": { input: &rayiov1alpha1.RayCluster{ ObjectMeta: metav1.ObjectMeta{ - Name: "ray-sample", + Name: "raycluster-sample", Namespace: "default", }, Spec: rayiov1alpha1.RayClusterSpec{ @@ -93,7 +93,7 @@ func TestGetHeadGroupServiceAccountName(t *testing.T) { "Ray cluster without head group service account": { input: &rayiov1alpha1.RayCluster{ ObjectMeta: metav1.ObjectMeta{ - Name: "ray-sample", + Name: "raycluster-sample", Namespace: "default", }, Spec: rayiov1alpha1.RayClusterSpec{ @@ -104,7 +104,7 @@ func TestGetHeadGroupServiceAccountName(t *testing.T) { }, }, }, - want: "ray-sample", + want: "raycluster-sample", }, } diff --git a/ray-operator/main.go b/ray-operator/main.go index 8900112f1c..0f25ea9ebd 100644 --- a/ray-operator/main.go +++ b/ray-operator/main.go @@ -16,7 +16,6 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/log/zap" - rayiov1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" // +kubebuilder:scaffold:imports ) @@ -31,7 +30,6 @@ var ( func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) - utilruntime.Must(rayiov1alpha1.AddToScheme(scheme)) utilruntime.Must(rayv1alpha1.AddToScheme(scheme)) // +kubebuilder:scaffold:scheme } diff --git a/tests/config/raycluster-service.yaml b/tests/config/raycluster-service.yaml index fbf7aab8b7..89a9cd5a41 100644 --- a/tests/config/raycluster-service.yaml +++ b/tests/config/raycluster-service.yaml @@ -5,7 +5,7 @@ metadata: spec: type: NodePort selector: - rayCluster: ray-compatibility-test + rayCluster: raycluster-compatibility-test ports: - name: redis port: 6379 From 15143b1110923e59f42cd34ef446dfed5231d045 Mon Sep 17 00:00:00 2001 From: brucez-anyscale Date: Tue, 24 May 2022 11:53:03 -0700 Subject: [PATCH 10/11] Revert naming --- ray-operator/controllers/ray/common/constant.go | 2 +- ray-operator/controllers/ray/raycluster_controller.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ray-operator/controllers/ray/common/constant.go b/ray-operator/controllers/ray/common/constant.go index c0f80ea92d..ca5c7704e5 100644 --- a/ray-operator/controllers/ray/common/constant.go +++ b/ray-operator/controllers/ray/common/constant.go @@ -8,7 +8,7 @@ const ( RayNodeLabelKey = "ray.io/is-ray-node" RayIDLabelKey = "ray.io/identifier" - // Use as separator for pod name, for example, ray-small-size-worker-0 + // Use as separator for pod name, for example, raycluster-small-size-worker-0 DashSymbol = "-" // Use as default port diff --git a/ray-operator/controllers/ray/raycluster_controller.go b/ray-operator/controllers/ray/raycluster_controller.go index b31007b6e3..2d33f46e65 100644 --- a/ray-operator/controllers/ray/raycluster_controller.go +++ b/ray-operator/controllers/ray/raycluster_controller.go @@ -511,9 +511,9 @@ func (r *RayClusterReconciler) buildWorkerPod(instance rayiov1alpha1.RayCluster, svcName := utils.GenerateServiceName(instance.Name) podTemplateSpec := common.DefaultWorkerPodTemplate(instance, worker, podName, svcName) pod := common.BuildPod(podTemplateSpec, rayiov1alpha1.WorkerNode, worker.RayStartParams, svcName) - // Set ray instance as the owner and controller + // Set raycluster instance as the owner and controller if err := controllerutil.SetControllerReference(&instance, &pod, r.Scheme); err != nil { - log.Error(err, "Failed to set controller reference for ray pod") + log.Error(err, "Failed to set controller reference for raycluster pod") } return pod From 5919f6d91c46c0c88db57e9a8e27f958c0283948 Mon Sep 17 00:00:00 2001 From: brucez-anyscale Date: Fri, 27 May 2022 21:16:27 -0700 Subject: [PATCH 11/11] Remove webhook --- .../apis/ray/v1alpha1/rayservice_types.go | 3 --- .../config/crd/bases/ray.io_rayservices.yaml | 5 ----- .../patches/cainjection_in_ray_rayservices.yaml | 7 ------- .../crd/patches/webhook_in_ray_rayservices.yaml | 16 ---------------- 4 files changed, 31 deletions(-) delete mode 100644 ray-operator/config/crd/patches/cainjection_in_ray_rayservices.yaml delete mode 100644 ray-operator/config/crd/patches/webhook_in_ray_rayservices.yaml diff --git a/ray-operator/apis/ray/v1alpha1/rayservice_types.go b/ray-operator/apis/ray/v1alpha1/rayservice_types.go index b993daf2f8..587fe5ea4c 100644 --- a/ray-operator/apis/ray/v1alpha1/rayservice_types.go +++ b/ray-operator/apis/ray/v1alpha1/rayservice_types.go @@ -11,9 +11,6 @@ import ( type RayServiceSpec struct { // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster // Important: Run "make" to regenerate code after modifying this file - - // Foo is an example field of RayService. Edit rayservice_types.go to remove/update - Foo string `json:"foo,omitempty"` } // RayServiceStatus defines the observed state of RayService diff --git a/ray-operator/config/crd/bases/ray.io_rayservices.yaml b/ray-operator/config/crd/bases/ray.io_rayservices.yaml index ba7ba26f7e..2a80b54f6b 100644 --- a/ray-operator/config/crd/bases/ray.io_rayservices.yaml +++ b/ray-operator/config/crd/bases/ray.io_rayservices.yaml @@ -33,11 +33,6 @@ spec: type: object spec: description: RayServiceSpec defines the desired state of RayService - properties: - foo: - description: Foo is an example field of RayService. Edit rayservice_types.go - to remove/update - type: string type: object status: description: RayServiceStatus defines the observed state of RayService diff --git a/ray-operator/config/crd/patches/cainjection_in_ray_rayservices.yaml b/ray-operator/config/crd/patches/cainjection_in_ray_rayservices.yaml deleted file mode 100644 index d46ece7891..0000000000 --- a/ray-operator/config/crd/patches/cainjection_in_ray_rayservices.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# The following patch adds a directive for certmanager to inject CA into the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) - name: rayservices.ray.io diff --git a/ray-operator/config/crd/patches/webhook_in_ray_rayservices.yaml b/ray-operator/config/crd/patches/webhook_in_ray_rayservices.yaml deleted file mode 100644 index 16a33aa850..0000000000 --- a/ray-operator/config/crd/patches/webhook_in_ray_rayservices.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# The following patch enables a conversion webhook for the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: rayservices.ray.io -spec: - conversion: - strategy: Webhook - webhook: - clientConfig: - service: - namespace: system - name: webhook-service - path: /convert - conversionReviewVersions: - - v1