From 95ace1a46b33ac92a23930d156ed3ee7adaa1c89 Mon Sep 17 00:00:00 2001 From: Patryk Strusiewicz-Surmacki Date: Tue, 16 Jan 2024 15:00:18 +0100 Subject: [PATCH 1/2] Added support for ClusterClass Co-authored-by: Lennart Jern Co-authored-by: Furkat Gofurov Signed-off-by: Patryk Strusiewicz-Surmacki --- Makefile | 25 ++ Tiltfile | 56 +++- api/v1beta1/metal3cluster_types.go | 2 +- api/v1beta1/metal3clustertemplate_types.go | 53 ++++ api/v1beta1/metal3clustertemplate_webhook.go | 67 +++++ .../metal3clustertemplate_webhook_test.go | 195 +++++++++++++ api/v1beta1/zz_generated.deepcopy.go | 90 ++++++ ...uster.x-k8s.io_metal3clustertemplates.yaml | 81 ++++++ config/crd/kustomization.yaml | 1 + config/webhook/manifests.yaml | 44 +++ examples/generate.sh | 34 ++- examples/templates/cluster.yaml | 255 +++++++++++++++++ examples/templates/clusterclass.yaml | 259 ++++++++++++++++++ hack/gen_tilt_settings.sh | 16 ++ main.go | 5 + 15 files changed, 1164 insertions(+), 19 deletions(-) create mode 100644 api/v1beta1/metal3clustertemplate_types.go create mode 100644 api/v1beta1/metal3clustertemplate_webhook.go create mode 100644 api/v1beta1/metal3clustertemplate_webhook_test.go create mode 100644 config/crd/bases/infrastructure.cluster.x-k8s.io_metal3clustertemplates.yaml create mode 100644 examples/templates/cluster.yaml create mode 100644 examples/templates/clusterclass.yaml diff --git a/Makefile b/Makefile index efb57edc2a..c03eb611c5 100644 --- a/Makefile +++ b/Makefile @@ -394,6 +394,10 @@ generate-manifests: $(CONTROLLER_GEN) ## Generate manifests e.g. CRD, RBAC etc. generate-examples: $(KUSTOMIZE) clean-examples ## Generate examples configurations to run a cluster. ./examples/generate.sh +.PHONY: generate-examples-clusterclass +generate-examples-clusterclass: $(KUSTOMIZE) clean-examples ## Generate examples configurations to run a cluster. + CLUSTER_TOPOLOGY=true ./examples/generate.sh + ## -------------------------------------- ## Docker ## -------------------------------------- @@ -471,18 +475,35 @@ deploy: generate-examples kubectl apply -f examples/_out/provider-components.yaml kubectl apply -f examples/_out/metal3crds.yaml +deploy-clusterclass: generate-examples-clusterclass + kubectl apply -f examples/_out/cert-manager.yaml + kubectl wait --for=condition=Available --timeout=300s -n cert-manager deployment cert-manager + kubectl wait --for=condition=Available --timeout=300s -n cert-manager deployment cert-manager-cainjector + kubectl wait --for=condition=Available --timeout=300s -n cert-manager deployment cert-manager-webhook + kubectl apply -f examples/_out/provider-components.yaml + kubectl apply -f examples/_out/metal3crds.yaml + deploy-examples: kubectl apply -f ./examples/_out/metal3plane.yaml kubectl apply -f ./examples/_out/cluster.yaml kubectl apply -f ./examples/_out/machinedeployment.yaml kubectl apply -f ./examples/_out/controlplane.yaml +deploy-examples-clusterclass: + kubectl apply -f ./examples/_out/metal3plane.yaml + kubectl apply -f ./examples/_out/clusterclass.yaml + kubectl apply -f ./examples/_out/cluster.yaml + delete-examples: kubectl delete -f ./examples/_out/machinedeployment.yaml || true kubectl delete -f ./examples/_out/controlplane.yaml || true kubectl delete -f ./examples/_out/cluster.yaml || true kubectl delete -f ./examples/_out/metal3plane.yaml || true +delete-examples-clusterclass: + kubectl delete -f ./examples/_out/cluster.yaml || true + kubectl delete -f ./examples/_out/clusterclass.yaml || true + kubectl delete -f ./examples/_out/metal3plane.yaml || true ## -------------------------------------- ## Release @@ -537,6 +558,10 @@ kind-create: ## create capm3 kind cluster if needed tilt-settings: ./hack/gen_tilt_settings.sh +.PHONY: tilt-settings-clusterclass +tilt-settings-clusterclass: + CLUSTER_TOPOLOGY=true ./hack/gen_tilt_settings.sh + .PHONY: tilt-up tilt-up: $(ENVSUBST) $(KUSTOMIZE) kind-create ## start tilt and build kind cluster if needed $(MAKE) deploy-bmo-crd diff --git a/Tiltfile b/Tiltfile index c706de54f4..a1934bfa55 100644 --- a/Tiltfile +++ b/Tiltfile @@ -69,27 +69,69 @@ def deploy_capi(): core_extra_args = extra_args.get("core") if core_extra_args: for namespace in ["capi-system", "capi-webhook-system"]: - patch_args_with_extra_args(namespace, "capi-controller-manager", core_extra_args) + patch_args_with_extra_args(namespace, "capi-controller-manager", core_extra_args, 1) patch_capi_manager_role_with_exp_infra_rbac() if extra_args.get("kubeadm-bootstrap"): kb_extra_args = extra_args.get("kubeadm-bootstrap") if kb_extra_args: - patch_args_with_extra_args("capi-kubeadm-bootstrap-system", "capi-kubeadm-bootstrap-controller-manager", kb_extra_args) - - -def patch_args_with_extra_args(namespace, name, extra_args): - args_str = str(local('kubectl get deployments {} -n {} -o jsonpath={{.spec.template.spec.containers[1].args}}'.format(name, namespace))) + patch_args_with_extra_args("capi-kubeadm-bootstrap-system", "capi-kubeadm-bootstrap-controller-manager", kb_extra_args, 1) + if extra_args.get("feature_gates"): + feature_gates = extra_args.get("feature_gates") + if feature_gates: + for gate in feature_gates: + set_feature_gate("capi-system", "capi-controller-manager", gate, feature_gates[gate], 0) + set_feature_gate("capi-kubeadm-control-plane-system", "capi-kubeadm-control-plane-controller-manager", gate, feature_gates[gate], 0) + +def patch_args_with_extra_args(namespace, name, extra_args, container): + args_str = str(local('kubectl get deployments {} -n {} -o jsonpath={{.spec.template.spec.containers[{}].args}}'.format(name, namespace, container))) args_to_add = [arg for arg in extra_args if arg not in args_str] if args_to_add: args = args_str[1:-1].split() args.extend(args_to_add) patch = [{ "op": "replace", - "path": "/spec/template/spec/containers/1/args", + "path": "/spec/template/spec/containers/" + str(container) + "/args", "value": args, }] local("kubectl patch deployment {} -n {} --type json -p='{}'".format(name, namespace, str(encode_json(patch)).replace("\n", ""))) +def set_feature_gate(namespace, name, feature_gate, value, container): + args_str = str(local('kubectl get deployments {} -n {} -o jsonpath={{.spec.template.spec.containers[{}].args}}'.format(name, namespace, container))) + args = args_str[2:-2].split("\",\"") + print("args") + print(args) + + args_to_add = [] + for arg in args: + print("arg") + print(arg) + lower_arg = arg.lower() + feature_gate_lower = feature_gate.lower() + start = lower_arg.find(feature_gate_lower) + if start > -1: + end = lower_arg.find(",", start) + if end < -1: + end = len(lower_arg)-1 + new_arg = arg[:start] + feature_gate + "=" + value + arg[end:] + print(new_arg) + args_to_add.append(new_arg) + else: + args_to_add.append(arg) + + print("args_to_add") + print(args_to_add) + if len(args_to_add) > 0: + patches = [] + for i in range(0, len(args_to_add)): + patch = [{ + "op": "replace", + "path": "/spec/template/spec/containers/" + str(container) + "/args/" + str(i), + "value": args_to_add[i], + }] + print("patch") + print(patch) + local("kubectl patch deployment {} -n {} --type json -p='{}'".format(name, namespace, str(encode_json(patch)).replace("\n", ""))) + # patch the CAPI manager role to also provide access to experimental infrastructure def patch_capi_manager_role_with_exp_infra_rbac(): diff --git a/api/v1beta1/metal3cluster_types.go b/api/v1beta1/metal3cluster_types.go index 365447f2ea..e91db43b8c 100644 --- a/api/v1beta1/metal3cluster_types.go +++ b/api/v1beta1/metal3cluster_types.go @@ -50,7 +50,7 @@ func (s *Metal3ClusterSpec) IsValid() error { } if s.ControlPlaneEndpoint.Port == 0 { - missing = append(missing, "ControlPlaneEndpoint.Host") + missing = append(missing, "ControlPlaneEndpoint.Port") } if len(missing) > 0 { diff --git a/api/v1beta1/metal3clustertemplate_types.go b/api/v1beta1/metal3clustertemplate_types.go new file mode 100644 index 0000000000..e096149499 --- /dev/null +++ b/api/v1beta1/metal3clustertemplate_types.go @@ -0,0 +1,53 @@ +/* +Copyright 2024 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// Metal3ClusterTemplateSpec defines the desired state of Metal3ClusterTemplate. +type Metal3ClusterTemplateSpec struct { + Template Metal3ClusterTemplateResource `json:"template"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:resource:path=metal3clustertemplates,scope=Namespaced,categories=cluster-api,shortName=m3ct +// +kubebuilder:storageversion + +// Metal3ClusterTemplate is the Schema for the metal3clustertemplates API. +type Metal3ClusterTemplate struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec Metal3ClusterTemplateSpec `json:"spec,omitempty"` +} + +// +kubebuilder:object:root=true + +// Metal3ClusterTemplateList contains a list of Metal3ClusterTemplate. +type Metal3ClusterTemplateList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []Metal3ClusterTemplate `json:"items"` +} + +func init() { + objectTypes = append(objectTypes, &Metal3ClusterTemplate{}, &Metal3ClusterTemplateList{}) +} + +// Metal3ClusterTemplateResource describes the data for creating a Metal3Cluster from a template. +type Metal3ClusterTemplateResource struct { + Spec Metal3ClusterSpec `json:"spec"` +} diff --git a/api/v1beta1/metal3clustertemplate_webhook.go b/api/v1beta1/metal3clustertemplate_webhook.go new file mode 100644 index 0000000000..b48edbd5a8 --- /dev/null +++ b/api/v1beta1/metal3clustertemplate_webhook.go @@ -0,0 +1,67 @@ +/* +Copyright 2024 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "github.com/pkg/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/webhook" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +// SetupWebhookWithManager sets up and registers the webhook with the manager. +func (c *Metal3ClusterTemplate) SetupWebhookWithManager(mgr ctrl.Manager) error { + return ctrl.NewWebhookManagedBy(mgr). + For(c). + Complete() +} + +// +kubebuilder:webhook:verbs=create;update,path=/validate-infrastructure-cluster-x-k8s-io-v1beta1-metal3clustertemplate,mutating=false,failurePolicy=fail,groups=infrastructure.cluster.x-k8s.io,resources=metal3clustertemplates,versions=v1beta1,name=validation.metal3clustertemplate.infrastructure.cluster.x-k8s.io,matchPolicy=Equivalent,sideEffects=None,admissionReviewVersions=v1;v1beta1 +// +kubebuilder:webhook:verbs=create;update,path=/mutate-infrastructure-cluster-x-k8s-io-v1beta1-metal3clustertemplate,mutating=true,failurePolicy=fail,groups=infrastructure.cluster.x-k8s.io,resources=metal3clustertemplates,versions=v1beta1,name=default.metal3clustertemplate.infrastructure.cluster.x-k8s.io,matchPolicy=Equivalent,sideEffects=None,admissionReviewVersions=v1;v1beta1 + +var _ webhook.Defaulter = &Metal3ClusterTemplate{} +var _ webhook.Validator = &Metal3ClusterTemplate{} + +func (c *Metal3ClusterTemplate) Default() { +} + +// ValidateCreate implements webhook.Validator so a webhook will be registered for the type. +func (c *Metal3ClusterTemplate) ValidateCreate() (admission.Warnings, error) { + return nil, c.validate() +} + +// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type. +func (c *Metal3ClusterTemplate) ValidateUpdate(old runtime.Object) (admission.Warnings, error) { + oldM3ct, ok := old.(*Metal3ClusterTemplate) + if !ok || oldM3ct == nil { + return nil, apierrors.NewInternalError(errors.New("unable to convert existing object")) + } + + if err := oldM3ct.Spec.Template.Spec.IsValid(); err != nil { + return nil, err + } + + return nil, c.validate() +} + +// ValidateDelete implements webhook.Validator so a webhook will be registered for the type. +func (c *Metal3ClusterTemplate) ValidateDelete() (admission.Warnings, error) { + return nil, nil +} + +func (c *Metal3ClusterTemplate) validate() error { + return c.Spec.Template.Spec.IsValid() +} diff --git a/api/v1beta1/metal3clustertemplate_webhook_test.go b/api/v1beta1/metal3clustertemplate_webhook_test.go new file mode 100644 index 0000000000..335a3cf18a --- /dev/null +++ b/api/v1beta1/metal3clustertemplate_webhook_test.go @@ -0,0 +1,195 @@ +/* +Copyright 2024 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "testing" + + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestMetal3ClusterTemplateDefault(t *testing.T) { + g := NewWithT(t) + + c := &Metal3ClusterTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "foo", + }, + Spec: Metal3ClusterTemplateSpec{}, + } + c.Default() + + g.Expect(c.Spec).To(Equal(Metal3ClusterTemplateSpec{})) +} + +func TestMetal3ClusterTemplateValidation(t *testing.T) { + tests := []struct { + name string + expectErr bool + c *Metal3ClusterTemplate + }{ + { + name: "should succeed when values and templates are correct", + expectErr: false, + c: &Metal3ClusterTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "foo", + }, + Spec: Metal3ClusterTemplateSpec{ + Template: Metal3ClusterTemplateResource{ + Spec: Metal3ClusterSpec{ + ControlPlaneEndpoint: APIEndpoint{ + Host: "127.0.0.1", + Port: 4242, + }, + }, + }, + }, + }, + }, + { + name: "should fail when values and templates are incorrect", + expectErr: true, + c: &Metal3ClusterTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "foo", + }, + Spec: Metal3ClusterTemplateSpec{ + Template: Metal3ClusterTemplateResource{ + Spec: Metal3ClusterSpec{ + ControlPlaneEndpoint: APIEndpoint{}, + }, + }, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + if tt.expectErr { + _, err := tt.c.ValidateCreate() + g.Expect(err).To(HaveOccurred()) + } else { + _, err := tt.c.ValidateCreate() + g.Expect(err).NotTo(HaveOccurred()) + } + _, err := tt.c.ValidateDelete() + g.Expect(err).NotTo(HaveOccurred()) + }) + } +} + +func TestMetal3ClusterTemplateUpdateValidation(t *testing.T) { + tests := []struct { + name string + expectErr bool + new *Metal3ClusterTemplateSpec + old *Metal3ClusterTemplateSpec + }{ + { + name: "should succeed when values and templates correct", + expectErr: false, + + new: &Metal3ClusterTemplateSpec{ + Template: Metal3ClusterTemplateResource{ + Spec: Metal3ClusterSpec{ + ControlPlaneEndpoint: APIEndpoint{ + Host: "127.0.0.1", + Port: 4242, + }, + }, + }, + }, + + old: &Metal3ClusterTemplateSpec{ + Template: Metal3ClusterTemplateResource{ + Spec: Metal3ClusterSpec{ + ControlPlaneEndpoint: APIEndpoint{ + Host: "127.0.0.1", + Port: 8080, + }, + }, + }, + }, + }, + { + name: "should fail if old template is invalid (e.g. missing host or port)", + expectErr: true, + + new: &Metal3ClusterTemplateSpec{ + Template: Metal3ClusterTemplateResource{ + Spec: Metal3ClusterSpec{ + ControlPlaneEndpoint: APIEndpoint{ + Host: "127.0.0.1", + Port: 8080, + }, + }, + }, + }, + old: &Metal3ClusterTemplateSpec{}, + }, + { + name: "should fail if new template is invalid (e.g. missing host or port)", + expectErr: true, + + new: &Metal3ClusterTemplateSpec{}, + old: &Metal3ClusterTemplateSpec{ + Template: Metal3ClusterTemplateResource{ + Spec: Metal3ClusterSpec{ + ControlPlaneEndpoint: APIEndpoint{ + Host: "127.0.0.1", + Port: 8080, + }, + }, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var newCT, oldCT *Metal3ClusterTemplate + g := NewWithT(t) + newCT = &Metal3ClusterTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "foo", + }, + Spec: *tt.new, + } + + if tt.old != nil { + oldCT = &Metal3ClusterTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "foo", + }, + Spec: *tt.old, + } + } else { + oldCT = nil + } + + if tt.expectErr { + _, err := newCT.ValidateUpdate(oldCT) + g.Expect(err).To(HaveOccurred()) + } else { + _, err := newCT.ValidateUpdate(oldCT) + g.Expect(err).NotTo(HaveOccurred()) + } + }) + } +} diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 0bf45443f2..3303a11940 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -449,6 +449,96 @@ func (in *Metal3ClusterStatus) DeepCopy() *Metal3ClusterStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Metal3ClusterTemplate) DeepCopyInto(out *Metal3ClusterTemplate) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Metal3ClusterTemplate. +func (in *Metal3ClusterTemplate) DeepCopy() *Metal3ClusterTemplate { + if in == nil { + return nil + } + out := new(Metal3ClusterTemplate) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Metal3ClusterTemplate) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Metal3ClusterTemplateList) DeepCopyInto(out *Metal3ClusterTemplateList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Metal3ClusterTemplate, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Metal3ClusterTemplateList. +func (in *Metal3ClusterTemplateList) DeepCopy() *Metal3ClusterTemplateList { + if in == nil { + return nil + } + out := new(Metal3ClusterTemplateList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Metal3ClusterTemplateList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Metal3ClusterTemplateResource) DeepCopyInto(out *Metal3ClusterTemplateResource) { + *out = *in + out.Spec = in.Spec +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Metal3ClusterTemplateResource. +func (in *Metal3ClusterTemplateResource) DeepCopy() *Metal3ClusterTemplateResource { + if in == nil { + return nil + } + out := new(Metal3ClusterTemplateResource) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Metal3ClusterTemplateSpec) DeepCopyInto(out *Metal3ClusterTemplateSpec) { + *out = *in + out.Template = in.Template +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Metal3ClusterTemplateSpec. +func (in *Metal3ClusterTemplateSpec) DeepCopy() *Metal3ClusterTemplateSpec { + if in == nil { + return nil + } + out := new(Metal3ClusterTemplateSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Metal3Data) DeepCopyInto(out *Metal3Data) { *out = *in diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_metal3clustertemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_metal3clustertemplates.yaml new file mode 100644 index 0000000000..bf3ee8f267 --- /dev/null +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_metal3clustertemplates.yaml @@ -0,0 +1,81 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.13.0 + name: metal3clustertemplates.infrastructure.cluster.x-k8s.io +spec: + group: infrastructure.cluster.x-k8s.io + names: + categories: + - cluster-api + kind: Metal3ClusterTemplate + listKind: Metal3ClusterTemplateList + plural: metal3clustertemplates + shortNames: + - m3ct + singular: metal3clustertemplate + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: Metal3ClusterTemplate is the Schema for the metal3clustertemplates + API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Metal3ClusterTemplateSpec defines the desired state of Metal3ClusterTemplate. + properties: + template: + description: Metal3ClusterTemplateResource describes the data for + creating a Metal3Cluster from a template. + properties: + spec: + description: Metal3ClusterSpec defines the desired state of Metal3Cluster. + properties: + controlPlaneEndpoint: + description: ControlPlaneEndpoint represents the endpoint + used to communicate with the control plane. + properties: + host: + description: Host is the hostname on which the API server + is serving. + type: string + port: + description: Port is the port on which the API server + is serving. + type: integer + required: + - host + - port + type: object + noCloudProvider: + description: Determines if the cluster is not to be deployed + with an external cloud provider. If set to true, CAPM3 will + use node labels to set providerID on the kubernetes nodes. + If set to false, providerID is set on nodes by other entities + and CAPM3 uses the value of the providerID on the m3m resource. + type: boolean + type: object + required: + - spec + type: object + required: + - template + type: object + type: object + served: true + storage: true diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 085d1581f2..07d6ba8c6c 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -9,6 +9,7 @@ commonLabels: resources: - bases/infrastructure.cluster.x-k8s.io_metal3clusters.yaml +- bases/infrastructure.cluster.x-k8s.io_metal3clustertemplates.yaml - bases/infrastructure.cluster.x-k8s.io_metal3machines.yaml - bases/infrastructure.cluster.x-k8s.io_metal3machinetemplates.yaml - bases/infrastructure.cluster.x-k8s.io_metal3datatemplates.yaml diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml index 85ee9f0db1..f0787b878a 100644 --- a/config/webhook/manifests.yaml +++ b/config/webhook/manifests.yaml @@ -26,6 +26,28 @@ webhooks: resources: - metal3clusters sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-metal3clustertemplate + failurePolicy: Fail + matchPolicy: Equivalent + name: default.metal3clustertemplate.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - metal3clustertemplates + sideEffects: None - admissionReviewVersions: - v1 - v1beta1 @@ -208,6 +230,28 @@ webhooks: resources: - metal3clusters sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /validate-infrastructure-cluster-x-k8s-io-v1beta1-metal3clustertemplate + failurePolicy: Fail + matchPolicy: Equivalent + name: validation.metal3clustertemplate.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - metal3clustertemplates + sideEffects: None - admissionReviewVersions: - v1 - v1beta1 diff --git a/examples/generate.sh b/examples/generate.sh index b13f19a6b6..964ac6f942 100755 --- a/examples/generate.sh +++ b/examples/generate.sh @@ -47,6 +47,7 @@ COMPONENTS_METAL3_GENERATED_FILE=${SOURCE_DIR}/provider-components/infrastructur PROVIDER_COMPONENTS_GENERATED_FILE=${OUTPUT_DIR}/provider-components.yaml CLUSTER_GENERATED_FILE=${OUTPUT_DIR}/cluster.yaml +CLUSTERCLASS_GENERATED_FILE=${OUTPUT_DIR}/clusterclass.yaml CONTROLPLANE_GENERATED_FILE=${OUTPUT_DIR}/controlplane.yaml MACHINEDEPLOYMENT_GENERATED_FILE=${OUTPUT_DIR}/machinedeployment.yaml METAL3PLANE_GENERATED_FILE=${OUTPUT_DIR}/metal3plane.yaml @@ -55,6 +56,7 @@ METAL3CRDS_GENERATED_FILE=${OUTPUT_DIR}/metal3crds.yaml # Overwrite flag. OVERWRITE=0 + SCRIPT=$(basename "$0") while test $# -gt 0; do case "$1" in @@ -94,13 +96,27 @@ ENVSUBST="${SOURCE_DIR}/envsubst-go" curl --fail -Ss -L -o "${ENVSUBST}" https://github.com/a8m/envsubst/releases/download/v1.2.0/envsubst-"$(uname -s)"-"$(uname -m)" chmod +x "$ENVSUBST" -# Generate cluster resources. -"$KUSTOMIZE" build "${SOURCE_DIR}/cluster" | "$ENVSUBST" >"${CLUSTER_GENERATED_FILE}" -echo "Generated ${CLUSTER_GENERATED_FILE}" - -# Generate controlplane resources. -"$KUSTOMIZE" build "${SOURCE_DIR}/controlplane" | "$ENVSUBST" >"${CONTROLPLANE_GENERATED_FILE}" -echo "Generated ${CONTROLPLANE_GENERATED_FILE}" +if [ -n "${CLUSTER_TOPOLOGY:-}" ]; then + # Generate clusterclass resources. + "$ENVSUBST" -i "${SOURCE_DIR}/templates/clusterclass.yaml" >"${CLUSTERCLASS_GENERATED_FILE}" + echo "Generated ${CLUSTERCLASS_GENERATED_FILE}" + + # Generate cluster resources. + "$ENVSUBST" -i "${SOURCE_DIR}/templates/cluster.yaml" >"${CLUSTER_GENERATED_FILE}" + echo "Generated ${CLUSTER_GENERATED_FILE}" +else + # Generate cluster resources. + "$KUSTOMIZE" build "${SOURCE_DIR}/cluster" | "$ENVSUBST" >"${CLUSTER_GENERATED_FILE}" + echo "Generated ${CLUSTER_GENERATED_FILE}" + + # Generate controlplane resources. + "$KUSTOMIZE" build "${SOURCE_DIR}/controlplane" | "$ENVSUBST" >"${CONTROLPLANE_GENERATED_FILE}" + echo "Generated ${CONTROLPLANE_GENERATED_FILE}" + + # Generate machinedeployment resources. + "$KUSTOMIZE" build "${SOURCE_DIR}/machinedeployment" | "$ENVSUBST" >>"${MACHINEDEPLOYMENT_GENERATED_FILE}" + echo "Generated ${MACHINEDEPLOYMENT_GENERATED_FILE}" +fi # Generate metal3crds resources. "$KUSTOMIZE" build "${SOURCE_DIR}/metal3crds" | "$ENVSUBST" >"${METAL3CRDS_GENERATED_FILE}" @@ -110,10 +126,6 @@ echo "Generated ${METAL3CRDS_GENERATED_FILE}" "$KUSTOMIZE" build "${SOURCE_DIR}/metal3plane" | "$ENVSUBST" >"${METAL3PLANE_GENERATED_FILE}" echo "Generated ${METAL3PLANE_GENERATED_FILE}" -# Generate machinedeployment resources. -"$KUSTOMIZE" build "${SOURCE_DIR}/machinedeployment" | "$ENVSUBST" >>"${MACHINEDEPLOYMENT_GENERATED_FILE}" -echo "Generated ${MACHINEDEPLOYMENT_GENERATED_FILE}" - # Get Cert-manager provider components file curl --fail -Ss -L -o "${COMPONENTS_CERT_MANAGER_GENERATED_FILE}" https://github.com/cert-manager/cert-manager/releases/download/v1.14.0/cert-manager.yaml echo "Downloaded ${COMPONENTS_CERT_MANAGER_GENERATED_FILE}" diff --git a/examples/templates/cluster.yaml b/examples/templates/cluster.yaml new file mode 100644 index 0000000000..94cb895fd5 --- /dev/null +++ b/examples/templates/cluster.yaml @@ -0,0 +1,255 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: Metal3DataTemplate +metadata: + name: example-cp-metadata +spec: + clusterName: ${CLUSTER_NAME} + metaData: + strings: + - key: abc + value: def + indexes: + - key: index_0_1 + offset: 0 + step: 1 + - key: index_3 + offset: 3 + - key: index_5_2 + offset: 5 + step: 2 + objectNames: + - key: machine_name + object: machine + - key: metal3machine_name + object: metal3machine + - key: bmh_name + object: baremetalhost + ipAddressesFromIPPool: + - key: ip_1 + name: pool1 + - key: ip_2 + name: pool2 + - key: ip6_1 + name: pool6-1 + - key: ip6_2 + name: pool6-2 + prefixesFromIPPool: + - key: prefix_1 + name: pool1 + - key: prefix_2 + name: pool2 + - key: prefix6_1 + name: pool6-1 + - key: prefix6_2 + name: pool6-2 + gatewaysFromIPPool: + - key: gateway_1 + name: pool1 + - key: gateway_2 + name: pool2 + - key: gateway6_1 + name: pool6-1 + - key: gateway6_2 + name: pool6-2 + fromHostInterfaces: + - key: mac + interface: eth0 + networkData: + links: + ethernets: + - type: phy + id: enp1s0 + macAddress: + fromHostInterface: eth0 + - type: phy + id: enp2s0 + macAddress: + fromHostInterface: eth1 + networks: + ipv4DHCP: + - id: provisioning + link: enp1s0 + ipv4: + - id: baremetal + link: enp2s0 + ipAddressFromIPPool: pool1 + routes: + - network: 0.0.0.0 + prefix: 0 + gateway: + fromIPPool: pool1 + services: + dns: + - 8.8.4.4 + services: + dns: + - 8.8.8.8 +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: Metal3DataTemplate +metadata: + name: example-md-metadata +spec: + clusterName: ${CLUSTER_NAME} + metaData: + strings: + - key: abc + value: def + indexes: + - key: index_0_1 + offset: 0 + step: 1 + - key: index_3 + offset: 3 + - key: index_5_2 + offset: 5 + step: 2 + objectNames: + - key: machine_name + object: machine + - key: metal3machine_name + object: metal3machine + - key: bmh_name + object: baremetalhost + ipAddressesFromIPPool: + - key: ip_1 + name: pool1 + - key: ip_2 + name: pool2 + - key: ip6_1 + name: pool6-1 + - key: ip6_2 + name: pool6-2 + prefixesFromIPPool: + - key: prefix_1 + name: pool1 + - key: prefix_2 + name: pool2 + - key: prefix6_1 + name: pool6-1 + - key: prefix6_2 + name: pool6-2 + gatewaysFromIPPool: + - key: gateway_1 + name: pool1 + - key: gateway_2 + name: pool2 + - key: gateway6_1 + name: pool6-1 + - key: gateway6_2 + name: pool6-2 + fromHostInterfaces: + - key: mac + interface: eth0 + networkData: + links: + ethernets: + - type: phy + id: enp1s0 + macAddress: + fromHostInterface: eth0 + - type: phy + id: enp2s0 + macAddress: + fromHostInterface: eth1 + networks: + ipv4DHCP: + - id: provisioning + link: enp1s0 + ipv4: + - id: baremetal + link: enp2s0 + ipAddressFromIPPool: pool1 + routes: + - network: 0.0.0.0 + prefix: 0 + gateway: + fromIPPool: pool1 + services: + dns: + - 8.8.4.4 + services: + dns: + - 8.8.8.8 +--- +apiVersion: ipam.metal3.io/v1alpha1 +kind: IPPool +metadata: + name: pool1 +spec: + clusterName: ${CLUSTER_NAME} + pools: + - start: 192.168.0.10 + end: 192.168.0.250 + prefix: 24 + gateway: 192.168.0.1 + namePrefix: pool1 +--- +apiVersion: ipam.metal3.io/v1alpha1 +kind: IPPool +metadata: + name: pool2 +spec: + clusterName: ${CLUSTER_NAME} + pools: + - subnet: 192.168.1.0/24 + prefix: 25 + gateway: 192.168.1.1 + namePrefix: pool2 +--- +apiVersion: ipam.metal3.io/v1alpha1 +kind: IPPool +metadata: + name: pool6-1 +spec: + clusterName: ${CLUSTER_NAME} + pools: + - start: 2001::10 + end: 2001::ff00 + prefix: 96 + gateway: 2001::1 + namePrefix: pool6-1 +--- +apiVersion: ipam.metal3.io/v1alpha1 +kind: IPPool +metadata: + name: pool6-2 +spec: + clusterName: ${CLUSTER_NAME} + pools: + - subnet: 2001:ABC::0/96 + prefix: 96 + gateway: 2001:ABC::1 + namePrefix: pool6-2 +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} + namespace: ${NAMESPACE} +spec: + topology: + class: example-clusterclass + version: ${KUBERNETES_VERSION} + controlPlane: + replicas: 1 + workers: + machineDeployments: + - class: example-worker + name: ${CLUSTER_NAME}-machine + replicas: 2 + variables: + - name: image + value: + checksum: ${IMAGE_CHECKSUM} + checksumType: ${IMAGE_CHECKSUM_TYPE} + format: ${IMAGE_FORMAT} + url: ${IMAGE_URL} + - name: controlPlaneEndpoint + value: + host: ${CLUSTER_APIENDPOINT_HOST} + port: ${CLUSTER_APIENDPOINT_PORT} + - name: workerDataTemplate + value: example-md-metadata + - name: controlPlaneDataTemplate + value: example-cp-metadata diff --git a/examples/templates/clusterclass.yaml b/examples/templates/clusterclass.yaml new file mode 100644 index 0000000000..184be93784 --- /dev/null +++ b/examples/templates/clusterclass.yaml @@ -0,0 +1,259 @@ +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: Metal3ClusterTemplate +metadata: + name: example-cluster-template +spec: + template: + spec: + controlPlaneEndpoint: + host: 127.0.0.1 + port: 6443 + noCloudProvider: true +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlaneTemplate +metadata: + name: example-controlplane +spec: + template: + spec: + kubeadmConfigSpec: + clusterConfiguration: {} + initConfiguration: + nodeRegistration: + name: '{{ ds.meta_data.local_hostname }}' + joinConfiguration: + controlPlane: {} + nodeRegistration: + name: '{{ ds.meta_data.local_hostname }}' +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: Metal3MachineTemplate +metadata: + name: example-md-0 +spec: + nodeReuse: false + template: + spec: + automatedCleaningMode: metadata + image: + checksum: ${IMAGE_CHECKSUM} + checksumType: ${IMAGE_CHECKSUM_TYPE} + format: ${IMAGE_FORMAT} + url: ${IMAGE_URL} + dataTemplate: + name: example-md-metadata +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: example-md-0 +spec: + template: + spec: + joinConfiguration: + nodeRegistration: + name: '{{ ds.meta_data.hostname }}' +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: Metal3MachineTemplate +metadata: + name: example-controlplane +spec: + nodeReuse: false + template: + spec: + automatedCleaningMode: metadata + image: + checksum: ${IMAGE_CHECKSUM} + checksumType: ${IMAGE_CHECKSUM_TYPE} + format: ${IMAGE_FORMAT} + url: ${IMAGE_URL} + dataTemplate: + name: example-cp-metadata +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: ClusterClass +metadata: + name: example-clusterclass +spec: + variables: + - name: controlPlaneEndpoint + required: true + schema: + openAPIV3Schema: + type: object + properties: + host: + type: string + port: + type: integer + - name: image + schema: + openAPIV3Schema: + type: object + properties: + checksum: + type: string + checksumType: + type: string + format: + type: string + url: + type: string + - name: workerDataTemplate + schema: + openAPIV3Schema: + type: string + - name: controlPlaneDataTemplate + schema: + openAPIV3Schema: + type: string + patches: + - name: controlPlaneEndpointSub + description: Overrides controlPlaneEndpoint data of Metal3ClusterTemplate used by the cluster + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: Metal3ClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: replace + path: /spec/template/spec/controlPlaneEndpoint + valueFrom: + variable: controlPlaneEndpoint + - name: imageSub + description: Overrides image data for worker nodes of example-worker class + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: Metal3MachineTemplate + matchResources: + machineDeploymentClass: + names: + - example-worker + jsonPatches: + - op: replace + path: /spec/template/spec/image/checksum + valueFrom: + variable: image.checksum + - op: replace + path: /spec/template/spec/image/checksumType + valueFrom: + variable: image.checksumType + - op: replace + path: /spec/template/spec/image/format + valueFrom: + variable: image.format + - op: replace + path: /spec/template/spec/image/url + valueFrom: + variable: image.url + - name: workerDataTemplateSub + description: Overrides data-template for worker nodes + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: Metal3MachineTemplate + matchResources: + machineDeploymentClass: + names: + - example-worker + jsonPatches: + - op: replace + path: /spec/template/spec/dataTemplate/name + valueFrom: + variable: workerDataTemplate + - name: controlPlaneImageSub + description: Overrides image data for worker nodes of control plane node + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: Metal3MachineTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: replace + path: /spec/template/spec/image/checksum + valueFrom: + variable: image.checksum + - op: replace + path: /spec/template/spec/image/checksumType + valueFrom: + variable: image.checksumType + - op: replace + path: /spec/template/spec/image/format + valueFrom: + variable: image.format + - op: replace + path: /spec/template/spec/image/url + valueFrom: + variable: image.url + - name: controlPlaneDataTemplateSub + description: Overrides data-template for control plane nodes + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: Metal3MachineTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: replace + path: /spec/template/spec/dataTemplate/name + valueFrom: + variable: controlPlaneDataTemplate + controlPlane: + ref: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlaneTemplate + name: example-controlplane + machineInfrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: Metal3MachineTemplate + name: example-md-0 + # This will create a MachineHealthCheck for ControlPlane machines. + machineHealthCheck: + nodeStartupTimeout: 3m + maxUnhealthy: 33% + unhealthyConditions: + - type: Ready + status: Unknown + timeout: 300s + - type: Ready + status: "False" + timeout: 300s + workers: + machineDeployments: + - class: example-worker + template: + metadata: + labels: + cluster.x-k8s.io/cluster-name: example + nodepool: nodepool-0 + bootstrap: + ref: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: example-md-0 + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: Metal3MachineTemplate + name: example-md-0 + # This will create a health check for each deployment created with the "test-worker" MachineDeploymentClass + machineHealthCheck: + unhealthyConditions: + - type: Ready + status: Unknown + timeout: 300s + - type: Ready + status: "False" + timeout: 300s + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: Metal3ClusterTemplate + name: example-cluster-template diff --git a/hack/gen_tilt_settings.sh b/hack/gen_tilt_settings.sh index 32126c3343..918a486940 100755 --- a/hack/gen_tilt_settings.sh +++ b/hack/gen_tilt_settings.sh @@ -44,6 +44,21 @@ get_latest_release() { CAPIRELEASEPATH="${CAPIRELEASEPATH:-https://api.github.com/repos/${CAPI_BASE_URL:-kubernetes-sigs/cluster-api}/releases}" export CAPIRELEASE="${CAPIRELEASE:-$(get_latest_release "${CAPIRELEASEPATH}" "v1.3.")}" +# ClusterClass enable flag +if [ -n "${CLUSTER_TOPOLOGY:-}" ]; then +cat <tilt-settings.json +{ + "capi_version": "${CAPIRELEASE}", + "cert_manager_version": "v1.12.3", + "kubernetes_version": "${KUBERNETES_VERSION:-v1.29.0}", + "extra_args": { + "feature_gates": { + "ClusterTopology": "true" + } + } +} +EOF +else cat <tilt-settings.json { "capi_version": "${CAPIRELEASE}", @@ -51,3 +66,4 @@ cat <tilt-settings.json "kubernetes_version": "${KUBERNETES_VERSION:-v1.30.0}" } EOF +fi diff --git a/main.go b/main.go index 78b3c05960..2428d20f93 100644 --- a/main.go +++ b/main.go @@ -493,6 +493,11 @@ func setupWebhooks(mgr ctrl.Manager) { setupLog.Error(err, "unable to create webhook", "webhook", "Metal3RemediationTemplate") os.Exit(1) } + + if err := (&infrav1.Metal3ClusterTemplate{}).SetupWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "Metal3ClusterTemplate") + os.Exit(1) + } } func concurrency(c int) controller.Options { From 7e3d56c74473bd6fc9013f539148469b9d6e0d44 Mon Sep 17 00:00:00 2001 From: Patryk Strusiewicz-Surmacki Date: Wed, 6 Mar 2024 01:10:17 +0100 Subject: [PATCH 2/2] Added basic e2e tests Signed-off-by: Patryk Strusiewicz-Surmacki --- Makefile | 35 +++ scripts/ci-e2e.sh | 8 +- test/e2e/config/e2e_conf.yaml | 2 + .../clusterclass-centos-kubeadm-config.yaml | 244 ++++++++++++++++++ .../kustomization.yaml | 4 + .../cluster-with-kcp.yaml | 119 +++++++++ .../bases/clusterclass-cluster/crs.yaml | 24 ++ .../clusterclass-cluster/kustomization.yaml | 4 + .../bases/clusterclass-cluster/md.yaml | 85 ++++++ .../clusterclass-ubuntu-kubeadm-config.yaml | 158 ++++++++++++ .../kustomization.yaml | 5 + .../bases/clusterclass/clusterclass.yaml | 165 ++++++++++++ .../bases/clusterclass/kustomization.yaml | 2 + .../kustomization.yaml | 3 + .../kustomization.yaml | 3 + .../kustomization.yaml | 2 + .../clusterclass/kustomization.yaml | 2 + 17 files changed, 864 insertions(+), 1 deletion(-) create mode 100644 test/e2e/data/infrastructure-metal3/bases/clusterclass-centos-kubeadm-config/clusterclass-centos-kubeadm-config.yaml create mode 100644 test/e2e/data/infrastructure-metal3/bases/clusterclass-centos-kubeadm-config/kustomization.yaml create mode 100644 test/e2e/data/infrastructure-metal3/bases/clusterclass-cluster/cluster-with-kcp.yaml create mode 100644 test/e2e/data/infrastructure-metal3/bases/clusterclass-cluster/crs.yaml create mode 100644 test/e2e/data/infrastructure-metal3/bases/clusterclass-cluster/kustomization.yaml create mode 100644 test/e2e/data/infrastructure-metal3/bases/clusterclass-cluster/md.yaml create mode 100644 test/e2e/data/infrastructure-metal3/bases/clusterclass-ubuntu-kubeadm-config/clusterclass-ubuntu-kubeadm-config.yaml create mode 100644 test/e2e/data/infrastructure-metal3/bases/clusterclass-ubuntu-kubeadm-config/kustomization.yaml create mode 100644 test/e2e/data/infrastructure-metal3/bases/clusterclass/clusterclass.yaml create mode 100644 test/e2e/data/infrastructure-metal3/bases/clusterclass/kustomization.yaml create mode 100644 test/e2e/data/infrastructure-metal3/clusterclass-template-centos/kustomization.yaml create mode 100644 test/e2e/data/infrastructure-metal3/clusterclass-template-ubuntu/kustomization.yaml create mode 100644 test/e2e/data/infrastructure-metal3/clusterclass-template-upgrade-workload/kustomization.yaml create mode 100644 test/e2e/data/infrastructure-metal3/clusterclass/kustomization.yaml diff --git a/Makefile b/Makefile index c03eb611c5..8eb3721c1d 100644 --- a/Makefile +++ b/Makefile @@ -140,6 +140,10 @@ test: lint unit ## Run tests test-e2e: ## Run e2e tests with capi e2e testing framework ./scripts/ci-e2e.sh +.PHONY: test-clusterclass-e2e +test-clusterclass-e2e: ## Run e2e tests with capi e2e testing framework + CLUSTER_TOPOLOGY=true GINKGO_FOCUS=basic ./scripts/ci-e2e.sh + GINKGO_NOCOLOR ?= false ARTIFACTS ?= $(ROOT_DIR)/_artifacts E2E_CONF_FILE ?= $(ROOT_DIR)/test/e2e/config/e2e_conf.yaml @@ -167,6 +171,14 @@ cluster-templates: $(KUSTOMIZE) ## Generate cluster templates $(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/cluster-template-ubuntu > $(E2E_OUT_DIR)/cluster-template-ubuntu.yaml $(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/cluster-template-centos > $(E2E_OUT_DIR)/cluster-template-centos.yaml $(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/cluster-template-upgrade-workload > $(E2E_OUT_DIR)/cluster-template-upgrade-workload.yaml + touch $(E2E_OUT_DIR)/clusterclass.yaml + +.PHONY: clusterclass-templates +clusterclass-templates: $(KUSTOMIZE) ## Generate cluster templates + $(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/clusterclass-template-ubuntu > $(E2E_OUT_DIR)/cluster-template-ubuntu.yaml + $(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/clusterclass-template-centos > $(E2E_OUT_DIR)/cluster-template-centos.yaml + $(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/clusterclass-template-upgrade-workload > $(E2E_OUT_DIR)/cluster-template-upgrade-workload.yaml + $(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/clusterclass > $(E2E_OUT_DIR)/clusterclass.yaml ## -------------------------------------- ## E2E Testing @@ -201,6 +213,29 @@ e2e-tests: $(GINKGO) e2e-substitutions cluster-templates # This target should be rm $(E2E_CONF_FILE_ENVSUBST) +.PHONY: e2e-clusterclass-tests +e2e-clusterclass-tests: CONTAINER_RUNTIME?=docker # Env variable can override this default +export CONTAINER_RUNTIME + +e2e-clusterclass-tests: $(GINKGO) e2e-substitutions clusterclass-templates # This target should be called from scripts/ci-e2e.sh + for image in $(E2E_CONTAINERS); do \ + $(CONTAINER_RUNTIME) pull $$image; \ + done + + $(GINKGO) --timeout=$(GINKGO_TIMEOUT) -v --trace --tags=e2e \ + --show-node-events --no-color=$(GINKGO_NOCOLOR) \ + --fail-fast="$(KEEP_TEST_ENV)" \ + --junit-report="junit.e2e_suite.1.xml" \ + --focus="$(GINKGO_FOCUS)" $(_SKIP_ARGS) "$(ROOT_DIR)/$(TEST_DIR)/e2e/" -- \ + -e2e.artifacts-folder="$(ARTIFACTS)" \ + -e2e.config="$(E2E_CONF_FILE_ENVSUBST)" \ + -e2e.skip-resource-cleanup=$(SKIP_CLEANUP) \ + -e2e.keep-test-environment=$(KEEP_TEST_ENV) \ + -e2e.trigger-ephemeral-test=$(EPHEMERAL_TEST) \ + -e2e.use-existing-cluster=$(SKIP_CREATE_MGMT_CLUSTER) + + rm $(E2E_CONF_FILE_ENVSUBST) + ## -------------------------------------- ## Build ## -------------------------------------- diff --git a/scripts/ci-e2e.sh b/scripts/ci-e2e.sh index 283eea1208..e4c4395150 100755 --- a/scripts/ci-e2e.sh +++ b/scripts/ci-e2e.sh @@ -80,4 +80,10 @@ source "${M3_DEV_ENV_PATH}/lib/ironic_tls_setup.sh" export LIVE_ISO_IMAGE="https://artifactory.nordix.org/artifactory/metal3/images/iso/minimal_linux_live-v2.iso" # run e2e tests -make e2e-tests +if [ -n "${CLUSTER_TOPOLOGY:-}" ]; then + export CLUSTER_TOPOLOGY=true + make e2e-clusterclass-tests +else + make e2e-tests +fi + diff --git a/test/e2e/config/e2e_conf.yaml b/test/e2e/config/e2e_conf.yaml index 38b5a9120a..9fcea4d52a 100644 --- a/test/e2e/config/e2e_conf.yaml +++ b/test/e2e/config/e2e_conf.yaml @@ -115,6 +115,8 @@ providers: targetName: "cluster-template-ubuntu.yaml" - sourcePath: "../_out/cluster-template-centos.yaml" targetName: "cluster-template-centos.yaml" + - sourcePath: "../_out/clusterclass.yaml" + targetName: "clusterclass-test-clusterclass.yaml" variables: CNI: "/tmp/calico.yaml" diff --git a/test/e2e/data/infrastructure-metal3/bases/clusterclass-centos-kubeadm-config/clusterclass-centos-kubeadm-config.yaml b/test/e2e/data/infrastructure-metal3/bases/clusterclass-centos-kubeadm-config/clusterclass-centos-kubeadm-config.yaml new file mode 100644 index 0000000000..712c4124b6 --- /dev/null +++ b/test/e2e/data/infrastructure-metal3/bases/clusterclass-centos-kubeadm-config/clusterclass-centos-kubeadm-config.yaml @@ -0,0 +1,244 @@ +--- +apiVersion: controlplane.cluster.x-k8s.io/${CAPI_VERSION} +kind: KubeadmControlPlaneTemplate +metadata: + name: ${CLUSTER_NAME} + namespace: ${NAMESPACE} +spec: +template: + spec: + kubeadmConfigSpec: + clusterConfiguration: {} + users: + - name: ${IMAGE_USERNAME} + sshAuthorizedKeys: + - ${SSH_PUB_KEY_CONTENT} + sudo: ALL=(ALL) NOPASSWD:ALL + files: + - content: | + #!/bin/bash + set -e + url="$1" + dst="$2" + filename="$(basename $url)" + tmpfile="/tmp/$filename" + curl -sSL -w "%{http_code}" "$url" | sed "s:/usr/bin:/usr/local/bin:g" > /tmp/"$filename" + http_status=$(cat "$tmpfile" | tail -n 1) + if [ "$http_status" != "200" ]; then + echo "Error: unable to retrieve $filename file"; + exit 1; + else + cat "$tmpfile"| sed '$d' > "$dst"; + fi + owner: root:root + path: /usr/local/bin/retrieve.configuration.files.sh + permissions: "0755" + - path: /etc/keepalived/keepalived.conf + content: | + ! Configuration File for keepalived + global_defs { + notification_email { + sysadmin@example.com + support@example.com + } + notification_email_from lb@example.com + smtp_server localhost + smtp_connect_timeout 30 + } + + vrrp_script k8s_api_check { + script "curl -sk https://127.0.0.1:6443/healthz" + interval 5 + timeout 5 + rise 3 + fall 3 + } + + vrrp_instance VI_1 { + state MASTER + interface eth1 + virtual_router_id 1 + priority 101 + advert_int 1 + virtual_ipaddress { + ${CLUSTER_APIENDPOINT_HOST} + } + track_script { + k8s_api_check + } + } + - path: /etc/NetworkManager/system-connections/eth0.nmconnection + owner: root:root + permissions: '0600' + content: | + [connection] + id=eth0 + type=ethernet + interface-name=eth0 + master=ironicendpoint + slave-type=bridge + - content: | + [connection] + id=ironicendpoint + type=bridge + interface-name=ironicendpoint + autoconnect=yes + autoconnect-priority=1 + + [bridge] + stp=false + interface-name=ironicendpoint + + [ipv4] + address1={{ ds.meta_data.provisioningIP }}/{{ ds.meta_data.provisioningCIDR }} + method=manual + + [ipv6] + addr-gen-mode=eui64 + method=ignore + path: /etc/NetworkManager/system-connections/ironicendpoint.nmconnection + owner: root:root + permissions: '0600' + - content: | + [kubernetes] + name=Kubernetes + baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64 + enabled=1 + gpgcheck=1 + repo_gpgcheck=0 + gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg + owner: root:root + path: /etc/yum.repos.d/kubernetes.repo + permissions: '0644' + - content: | + [registries.search] + registries = ['docker.io'] + + [registries.insecure] + registries = ['${REGISTRY}'] + path: /etc/containers/registries.conf + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + cgroup-driver: systemd + container-runtime-endpoint: unix:///var/run/crio/crio.sock + feature-gates: AllAlpha=false + node-labels: metal3.io/uuid={{ ds.meta_data.uuid }} + provider-id: ${PROVIDER_ID_FORMAT} + runtime-request-timeout: 5m + name: '{{ ds.meta_data.name }}' + joinConfiguration: + controlPlane: {} + nodeRegistration: + kubeletExtraArgs: + cgroup-driver: systemd + container-runtime-endpoint: unix:///var/run/crio/crio.sock + feature-gates: AllAlpha=false + node-labels: metal3.io/uuid={{ ds.meta_data.uuid }} + provider-id: ${PROVIDER_ID_FORMAT} + runtime-request-timeout: 5m + name: '{{ ds.meta_data.name }}' + preKubeadmCommands: + - rm /etc/cni/net.d/* + - systemctl restart NetworkManager.service + - nmcli connection load /etc/NetworkManager/system-connections/ironicendpoint.nmconnection + - nmcli connection up ironicendpoint + - nmcli connection load /etc/NetworkManager/system-connections/eth0.nmconnection + - nmcli connection up eth0 + - systemctl enable --now keepalived + - sleep 60 + - systemctl enable --now crio kubelet + postKubeadmCommands: + - mkdir -p /home/${IMAGE_USERNAME}/.kube + - chown ${IMAGE_USERNAME}:${IMAGE_USERNAME} /home/${IMAGE_USERNAME}/.kube + - cp /etc/kubernetes/admin.conf /home/${IMAGE_USERNAME}/.kube/config + - chown ${IMAGE_USERNAME}:${IMAGE_USERNAME} /home/${IMAGE_USERNAME}/.kube/config + +--- +apiVersion: bootstrap.cluster.x-k8s.io/${CAPI_VERSION} +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-workers + namespace: ${NAMESPACE} +spec: + template: + spec: + files: + - content: | + #!/bin/bash + set -e + url="$1" + dst="$2" + filename="$(basename $url)" + tmpfile="/tmp/$filename" + curl -sSL -w "%{http_code}" "$url" | sed "s:/usr/bin:/usr/local/bin:g" > /tmp/"$filename" + http_status=$(cat "$tmpfile" | tail -n 1) + if [ "$http_status" != "200" ]; then + echo "Error: unable to retrieve $filename file"; + exit 1; + else + cat "$tmpfile"| sed '$d' > "$dst"; + fi + owner: root:root + path: /usr/local/bin/retrieve.configuration.files.sh + permissions: "0755" + - content: | + [connection] + id=eth0 + type=ethernet + interface-name=eth0 + master=ironicendpoint + slave-type=bridge + autoconnect=yes + autoconnect-priority=999 + path: /etc/NetworkManager/system-connections/eth0.nmconnection + owner: root:root + permissions: '0600' + - path: /etc/NetworkManager/system-connections/ironicendpoint.nmconnection + owner: root:root + permissions: '0600' + content: | + [connection] + id=ironicendpoint + type=bridge + interface-name=ironicendpoint + + [bridge] + stp=false + + [ipv4] + address1={{ ds.meta_data.provisioningIP }}/{{ ds.meta_data.provisioningCIDR }} + method=manual + + [ipv6] + addr-gen-mode=eui64 + method=ignore + - path: /etc/yum.repos.d/kubernetes.repo + owner: root:root + permissions: '0644' + content: | + [kubernetes] + name=Kubernetes + baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64 + enabled=1 + gpgcheck=1 + repo_gpgcheck=0 + gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg + - path : /etc/containers/registries.conf + owner: root:root + permissions: '0644' + content: | + [registries.search] + registries = ['docker.io'] + + [registries.insecure] + registries = ['${REGISTRY}'] + preKubeadmCommands: + - rm /etc/cni/net.d/* + - systemctl restart NetworkManager.service + - nmcli connection load /etc/NetworkManager/system-connections/ironicendpoint.nmconnection + - nmcli connection up ironicendpoint + - nmcli connection load /etc/NetworkManager/system-connections/eth0.nmconnection + - nmcli connection up eth0 + - systemctl enable --now crio kubelet + - sleep 120 diff --git a/test/e2e/data/infrastructure-metal3/bases/clusterclass-centos-kubeadm-config/kustomization.yaml b/test/e2e/data/infrastructure-metal3/bases/clusterclass-centos-kubeadm-config/kustomization.yaml new file mode 100644 index 0000000000..9afce72768 --- /dev/null +++ b/test/e2e/data/infrastructure-metal3/bases/clusterclass-centos-kubeadm-config/kustomization.yaml @@ -0,0 +1,4 @@ +resources: +- ../clusterclass-cluster +patchesStrategicMerge: + - clusterclass-centos-kubeadm-config.yaml \ No newline at end of file diff --git a/test/e2e/data/infrastructure-metal3/bases/clusterclass-cluster/cluster-with-kcp.yaml b/test/e2e/data/infrastructure-metal3/bases/clusterclass-cluster/cluster-with-kcp.yaml new file mode 100644 index 0000000000..620ffef6a3 --- /dev/null +++ b/test/e2e/data/infrastructure-metal3/bases/clusterclass-cluster/cluster-with-kcp.yaml @@ -0,0 +1,119 @@ +apiVersion: controlplane.cluster.x-k8s.io/${CAPI_VERSION} +kind: KubeadmControlPlaneTemplate +metadata: + name: ${CLUSTER_NAME} + namespace: ${NAMESPACE} +--- +apiVersion: infrastructure.cluster.x-k8s.io/${CAPM3_VERSION} +kind: Metal3ClusterTemplate +metadata: + name: ${CLUSTER_NAME} + namespace: ${NAMESPACE} +spec: + template: + spec: + controlPlaneEndpoint: + host: ${CLUSTER_APIENDPOINT_HOST} + port: ${CLUSTER_APIENDPOINT_PORT} + noCloudProvider: true +--- +apiVersion: infrastructure.cluster.x-k8s.io/${CAPM3_VERSION} +kind: Metal3MachineTemplate +metadata: + name: ${CLUSTER_NAME}-controlplane + namespace: ${NAMESPACE} +spec: + template: + spec: + dataTemplate: + name: ${CLUSTER_NAME}-controlplane-template + image: + checksum: ${IMAGE_RAW_CHECKSUM} + checksumType: ${IMAGE_CHECKSUM_TYPE} + format: raw + url: ${IMAGE_RAW_URL} +--- +apiVersion: infrastructure.cluster.x-k8s.io/${CAPM3_VERSION} +kind: Metal3DataTemplate +metadata: + name: ${CLUSTER_NAME}-controlplane-template + namespace: ${NAMESPACE} +spec: + clusterName: ${CLUSTER_NAME} + metaData: + ipAddressesFromIPPool: + - key: provisioningIP + name: provisioning-pool + objectNames: + - key: name + object: machine + - key: local-hostname + object: machine + - key: local_hostname + object: machine + prefixesFromIPPool: + - key: provisioningCIDR + name: provisioning-pool + networkData: + links: + ethernets: + - id: enp1s0 + macAddress: + fromHostInterface: enp1s0 + type: phy + - id: enp2s0 + macAddress: + fromHostInterface: enp2s0 + type: phy + networks: + ipv4: + - id: baremetalv4 + ipAddressFromIPPool: baremetalv4-pool + link: enp2s0 + routes: + - gateway: + fromIPPool: baremetalv4-pool + network: 0.0.0.0 + prefix: 0 + services: + dns: + - 8.8.8.8 +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + cni: ${CLUSTER_NAME}-crs-0 + name: ${CLUSTER_NAME} + namespace: ${NAMESPACE} +spec: + clusterNetwork: + pods: + cidrBlocks: ["${POD_CIDR}"] + services: + cidrBlocks: ["${SERVICE_CIDR}"] + topology: + class: test-clusterclass + version: ${KUBERNETES_VERSION} + controlPlane: + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + workers: + machineDeployments: + - class: worker + name: ${CLUSTER_NAME}-machine + replicas: ${WORKER_MACHINE_COUNT} + variables: + - name: image + value: + checksum: ${IMAGE_RAW_CHECKSUM} + checksumType: ${IMAGE_CHECKSUM_TYPE} + format: raw + url: ${IMAGE_RAW_URL} + - name: controlPlaneEndpoint + value: + host: ${CLUSTER_APIENDPOINT_HOST} + port: ${CLUSTER_APIENDPOINT_PORT} + - name: workerDataTemplate + value: ${CLUSTER_NAME}-workers-template + - name: controlPlaneDataTemplate + value: ${CLUSTER_NAME}-controlplane-template diff --git a/test/e2e/data/infrastructure-metal3/bases/clusterclass-cluster/crs.yaml b/test/e2e/data/infrastructure-metal3/bases/clusterclass-cluster/crs.yaml new file mode 100644 index 0000000000..559a5cefee --- /dev/null +++ b/test/e2e/data/infrastructure-metal3/bases/clusterclass-cluster/crs.yaml @@ -0,0 +1,24 @@ +--- +# ConfigMap object referenced by the ClusterResourceSet object and with +# the CNI resource defined in the test config file +apiVersion: v1 +kind: ConfigMap +metadata: + name: "cni-${CLUSTER_NAME}-crs-0" +data: ${CNI_RESOURCES} +binaryData: +--- +# ClusterResourceSet object with +# a selector that targets all the Cluster with label cni=${CLUSTER_NAME}-crs-0 +apiVersion: addons.cluster.x-k8s.io/${CAPI_VERSION} +kind: ClusterResourceSet +metadata: + name: "${CLUSTER_NAME}-crs-0" +spec: + strategy: ApplyOnce + clusterSelector: + matchLabels: + cni: "${CLUSTER_NAME}-crs-0" + resources: + - name: "cni-${CLUSTER_NAME}-crs-0" + kind: ConfigMap diff --git a/test/e2e/data/infrastructure-metal3/bases/clusterclass-cluster/kustomization.yaml b/test/e2e/data/infrastructure-metal3/bases/clusterclass-cluster/kustomization.yaml new file mode 100644 index 0000000000..52b746df63 --- /dev/null +++ b/test/e2e/data/infrastructure-metal3/bases/clusterclass-cluster/kustomization.yaml @@ -0,0 +1,4 @@ +resources: +- cluster-with-kcp.yaml +- md.yaml +- crs.yaml diff --git a/test/e2e/data/infrastructure-metal3/bases/clusterclass-cluster/md.yaml b/test/e2e/data/infrastructure-metal3/bases/clusterclass-cluster/md.yaml new file mode 100644 index 0000000000..be01049aba --- /dev/null +++ b/test/e2e/data/infrastructure-metal3/bases/clusterclass-cluster/md.yaml @@ -0,0 +1,85 @@ +apiVersion: infrastructure.cluster.x-k8s.io/${CAPM3_VERSION} +kind: Metal3MachineTemplate +metadata: + name: ${CLUSTER_NAME}-workers + namespace: ${NAMESPACE} +spec: + template: + spec: + dataTemplate: + name: ${CLUSTER_NAME}-workers-template + image: + checksum: ${IMAGE_RAW_CHECKSUM} + checksumType: ${IMAGE_CHECKSUM_TYPE} + format: raw + url: ${IMAGE_RAW_URL} +--- +apiVersion: infrastructure.cluster.x-k8s.io/${CAPM3_VERSION} +kind: Metal3DataTemplate +metadata: + name: ${CLUSTER_NAME}-workers-template + namespace: ${NAMESPACE} +spec: + clusterName: ${CLUSTER_NAME} + metaData: + ipAddressesFromIPPool: + - key: provisioningIP + name: provisioning-pool + objectNames: + - key: name + object: machine + - key: local-hostname + object: machine + - key: local_hostname + object: machine + prefixesFromIPPool: + - key: provisioningCIDR + name: provisioning-pool + networkData: + links: + ethernets: + - id: enp1s0 + macAddress: + fromHostInterface: enp1s0 + type: phy + - id: enp2s0 + macAddress: + fromHostInterface: enp2s0 + type: phy + networks: + ipv4: + - id: baremetalv4 + ipAddressFromIPPool: baremetalv4-pool + link: enp2s0 + routes: + - gateway: + fromIPPool: baremetalv4-pool + network: 0.0.0.0 + prefix: 0 + services: + dns: + - 8.8.8.8 +--- +apiVersion: bootstrap.cluster.x-k8s.io/${CAPI_VERSION} +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-workers + namespace: ${NAMESPACE} +spec: + template: + spec: + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cgroup-driver: systemd + container-runtime-endpoint: unix:///var/run/crio/crio.sock + feature-gates: AllAlpha=false + node-labels: metal3.io/uuid={{ ds.meta_data.uuid }} + provider-id: ${PROVIDER_ID_FORMAT} + runtime-request-timeout: 5m + name: '{{ ds.meta_data.name }}' + users: + - name: metal3 + sshAuthorizedKeys: + - ${SSH_PUB_KEY_CONTENT} + sudo: ALL=(ALL) NOPASSWD:ALL diff --git a/test/e2e/data/infrastructure-metal3/bases/clusterclass-ubuntu-kubeadm-config/clusterclass-ubuntu-kubeadm-config.yaml b/test/e2e/data/infrastructure-metal3/bases/clusterclass-ubuntu-kubeadm-config/clusterclass-ubuntu-kubeadm-config.yaml new file mode 100644 index 0000000000..dfadf31855 --- /dev/null +++ b/test/e2e/data/infrastructure-metal3/bases/clusterclass-ubuntu-kubeadm-config/clusterclass-ubuntu-kubeadm-config.yaml @@ -0,0 +1,158 @@ +--- +apiVersion: controlplane.cluster.x-k8s.io/${CAPI_VERSION} +kind: KubeadmControlPlaneTemplate +metadata: + name: ${CLUSTER_NAME} + namespace: ${NAMESPACE} +spec: + template: + spec: + kubeadmConfigSpec: + clusterConfiguration: {} + users: + - name: ${IMAGE_USERNAME} + sshAuthorizedKeys: + - ${SSH_PUB_KEY_CONTENT} + sudo: ALL=(ALL) NOPASSWD:ALL + files: + - content: | + #!/bin/bash + while :; do + curl -sk https://127.0.0.1:6443/healthz 1>&2 > /dev/null + isOk=$? + isActive=$(systemctl show -p ActiveState keepalived.service | cut -d'=' -f2) + if [ $isOk == "0" ] && [ $isActive != "active" ]; then + logger 'API server is healthy, however keepalived is not running, starting keepalived' + echo 'API server is healthy, however keepalived is not running, starting keepalived' + sudo systemctl start keepalived.service + elif [ $isOk != "0" ] && [ $isActive == "active" ]; then + logger 'API server is not healthy, however keepalived running, stopping keepalived' + echo 'API server is not healthy, however keepalived running, stopping keepalived' + sudo systemctl stop keepalived.service + fi + sleep 5 + done + owner: root:root + path: /usr/local/bin/monitor.keepalived.sh + permissions: "0755" + - content: | + [Unit] + Description=Monitors keepalived adjusts status with that of API server + After=syslog.target network-online.target + + [Service] + Type=simple + Restart=always + ExecStart=/usr/local/bin/monitor.keepalived.sh + + [Install] + WantedBy=multi-user.target + owner: root:root + path: /lib/systemd/system/monitor.keepalived.service + - content: | + ! Configuration File for keepalived + global_defs { + notification_email { + sysadmin@example.com + support@example.com + } + notification_email_from lb@example.com + smtp_server localhost + smtp_connect_timeout 30 + } + vrrp_instance VI_2 { + state MASTER + interface enp2s0 + virtual_router_id 2 + priority 101 + advert_int 1 + virtual_ipaddress { + ${CLUSTER_APIENDPOINT_HOST} + } + } + path: /etc/keepalived/keepalived.conf + - content: | + network: + version: 2 + renderer: networkd + bridges: + ${CLUSTER_PROVISIONING_INTERFACE}: + interfaces: [enp1s0] + addresses: + - {{ ds.meta_data.provisioningIP }}/{{ ds.meta_data.provisioningCIDR }} + owner: root:root + path: /etc/netplan/52-ironicendpoint.yaml + permissions: "0644" + - content: | + [registries.search] + registries = ['docker.io'] + + [registries.insecure] + registries = ['${REGISTRY}'] + path: /etc/containers/registries.conf + joinConfiguration: + controlPlane: {} + nodeRegistration: + kubeletExtraArgs: + cgroup-driver: systemd + container-runtime-endpoint: unix:///var/run/crio/crio.sock + feature-gates: AllAlpha=false + node-labels: metal3.io/uuid={{ ds.meta_data.uuid }} + provider-id: ${PROVIDER_ID_FORMAT} + runtime-request-timeout: 5m + name: '{{ ds.meta_data.name }}' + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + cgroup-driver: systemd + container-runtime-endpoint: unix:///var/run/crio/crio.sock + feature-gates: AllAlpha=false + node-labels: metal3.io/uuid={{ ds.meta_data.uuid }} + provider-id: ${PROVIDER_ID_FORMAT} + runtime-request-timeout: 5m + name: '{{ ds.meta_data.name }}' + postKubeadmCommands: + - mkdir -p /home/${IMAGE_USERNAME}/.kube + - chown ${IMAGE_USERNAME}:${IMAGE_USERNAME} /home/${IMAGE_USERNAME}/.kube + - cp /etc/kubernetes/admin.conf /home/${IMAGE_USERNAME}/.kube/config + - systemctl enable --now keepalived + - chown ${IMAGE_USERNAME}:${IMAGE_USERNAME} /home/${IMAGE_USERNAME}/.kube/config + preKubeadmCommands: + - sed -i "s/MACAddressPolicy=persistent/MACAddressPolicy=none/g" /usr/lib/systemd/network/99-default.link + - netplan apply + - systemctl enable --now crio kubelet + - if (curl -sk --max-time 10 https://${CLUSTER_APIENDPOINT_HOST}:${CLUSTER_APIENDPOINT_PORT}/healthz); then echo "keepalived already running";else systemctl start keepalived; fi + - systemctl enable --now /lib/systemd/system/monitor.keepalived.service +--- +apiVersion: bootstrap.cluster.x-k8s.io/${CAPI_VERSION} +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-workers + namespace: ${NAMESPACE} +spec: + template: + spec: + files: + - content: | + network: + version: 2 + renderer: networkd + bridges: + ${CLUSTER_PROVISIONING_INTERFACE}: + interfaces: [enp1s0] + addresses: + - {{ ds.meta_data.provisioningIP }}/{{ ds.meta_data.provisioningCIDR }} + owner: root:root + path: /etc/netplan/52-ironicendpoint.yaml + permissions: "0644" + - content: | + [registries.search] + registries = ['docker.io'] + + [registries.insecure] + registries = ['${REGISTRY}'] + path: /etc/containers/registries.conf + preKubeadmCommands: + - sed -i "s/MACAddressPolicy=persistent/MACAddressPolicy=none/g" /usr/lib/systemd/network/99-default.link + - netplan apply + - systemctl enable --now crio kubelet diff --git a/test/e2e/data/infrastructure-metal3/bases/clusterclass-ubuntu-kubeadm-config/kustomization.yaml b/test/e2e/data/infrastructure-metal3/bases/clusterclass-ubuntu-kubeadm-config/kustomization.yaml new file mode 100644 index 0000000000..d46c116191 --- /dev/null +++ b/test/e2e/data/infrastructure-metal3/bases/clusterclass-ubuntu-kubeadm-config/kustomization.yaml @@ -0,0 +1,5 @@ + +resources: +- ../clusterclass-cluster +patchesStrategicMerge: + - clusterclass-ubuntu-kubeadm-config.yaml \ No newline at end of file diff --git a/test/e2e/data/infrastructure-metal3/bases/clusterclass/clusterclass.yaml b/test/e2e/data/infrastructure-metal3/bases/clusterclass/clusterclass.yaml new file mode 100644 index 0000000000..3bf1d2da43 --- /dev/null +++ b/test/e2e/data/infrastructure-metal3/bases/clusterclass/clusterclass.yaml @@ -0,0 +1,165 @@ +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: ClusterClass +metadata: + name: test-clusterclass +spec: + variables: + - name: controlPlaneEndpoint + required: true + schema: + openAPIV3Schema: + type: object + properties: + host: + type: string + port: + type: integer + - name: image + schema: + openAPIV3Schema: + type: object + properties: + checksum: + type: string + checksumType: + type: string + format: + type: string + url: + type: string + - name: workerDataTemplate + schema: + openAPIV3Schema: + type: string + - name: controlPlaneDataTemplate + schema: + openAPIV3Schema: + type: string + patches: + - name: controlPlaneEndpointSub + description: Overrides controlPlaneEndpoint data of Metal3ClusterTemplate used by the cluster + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: Metal3ClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: replace + path: /spec/template/spec/controlPlaneEndpoint + valueFrom: + variable: controlPlaneEndpoint + - name: imageSub + description: Overrides image data for worker nodes of example-worker class + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: Metal3MachineTemplate + matchResources: + machineDeploymentClass: + names: + - worker + jsonPatches: + - op: replace + path: /spec/template/spec/image/checksum + valueFrom: + variable: image.checksum + - op: replace + path: /spec/template/spec/image/checksumType + valueFrom: + variable: image.checksumType + - op: replace + path: /spec/template/spec/image/format + valueFrom: + variable: image.format + - op: replace + path: /spec/template/spec/image/url + valueFrom: + variable: image.url + - name: workerDataTemplateSub + description: Overrides data-template for worker nodes + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: Metal3MachineTemplate + matchResources: + machineDeploymentClass: + names: + - worker + jsonPatches: + - op: replace + path: /spec/template/spec/dataTemplate/name + valueFrom: + variable: workerDataTemplate + - name: controlPlaneImageSub + description: Overrides image data for worker nodes of control plane node + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: Metal3MachineTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: replace + path: /spec/template/spec/image/checksum + valueFrom: + variable: image.checksum + - op: replace + path: /spec/template/spec/image/checksumType + valueFrom: + variable: image.checksumType + - op: replace + path: /spec/template/spec/image/format + valueFrom: + variable: image.format + - op: replace + path: /spec/template/spec/image/url + valueFrom: + variable: image.url + - name: controlPlaneDataTemplateSub + description: Overrides data-template for control plane nodes + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: Metal3MachineTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: replace + path: /spec/template/spec/dataTemplate/name + valueFrom: + variable: controlPlaneDataTemplate + controlPlane: + ref: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlaneTemplate + name: ${CLUSTER_NAME} + machineInfrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: Metal3MachineTemplate + name: ${CLUSTER_NAME}-controlplane + workers: + machineDeployments: + - class: worker + template: + metadata: + labels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + nodepool: nodepool-0 + bootstrap: + ref: + apiVersion: bootstrap.cluster.x-k8s.io/${CAPI_VERSION} + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-workers + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/${CAPM3_VERSION} + kind: Metal3MachineTemplate + name: ${CLUSTER_NAME}-workers + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: Metal3ClusterTemplate + name: ${CLUSTER_NAME} \ No newline at end of file diff --git a/test/e2e/data/infrastructure-metal3/bases/clusterclass/kustomization.yaml b/test/e2e/data/infrastructure-metal3/bases/clusterclass/kustomization.yaml new file mode 100644 index 0000000000..6fe2104ddf --- /dev/null +++ b/test/e2e/data/infrastructure-metal3/bases/clusterclass/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- clusterclass.yaml diff --git a/test/e2e/data/infrastructure-metal3/clusterclass-template-centos/kustomization.yaml b/test/e2e/data/infrastructure-metal3/clusterclass-template-centos/kustomization.yaml new file mode 100644 index 0000000000..0d7e2f7b2b --- /dev/null +++ b/test/e2e/data/infrastructure-metal3/clusterclass-template-centos/kustomization.yaml @@ -0,0 +1,3 @@ +resources: +- ../bases/ippool +- ../bases/clusterclass-centos-kubeadm-config diff --git a/test/e2e/data/infrastructure-metal3/clusterclass-template-ubuntu/kustomization.yaml b/test/e2e/data/infrastructure-metal3/clusterclass-template-ubuntu/kustomization.yaml new file mode 100644 index 0000000000..74b543141f --- /dev/null +++ b/test/e2e/data/infrastructure-metal3/clusterclass-template-ubuntu/kustomization.yaml @@ -0,0 +1,3 @@ +resources: +- ../bases/ippool +- ../bases/clusterclass-ubuntu-kubeadm-config \ No newline at end of file diff --git a/test/e2e/data/infrastructure-metal3/clusterclass-template-upgrade-workload/kustomization.yaml b/test/e2e/data/infrastructure-metal3/clusterclass-template-upgrade-workload/kustomization.yaml new file mode 100644 index 0000000000..81cd049af0 --- /dev/null +++ b/test/e2e/data/infrastructure-metal3/clusterclass-template-upgrade-workload/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- ../bases/clusterclass-ubuntu-kubeadm-config diff --git a/test/e2e/data/infrastructure-metal3/clusterclass/kustomization.yaml b/test/e2e/data/infrastructure-metal3/clusterclass/kustomization.yaml new file mode 100644 index 0000000000..19d54e0d1d --- /dev/null +++ b/test/e2e/data/infrastructure-metal3/clusterclass/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- ../bases/clusterclass