Adding cluster set types

Adds types for cluster sets, which allows constraining a few elements of
clusters including: overall resource usage, and which nodes it can use.

charts/k3k/crds/k3k.io_clusters.yaml

@@ -77,6 +77,31 @@ spec:
                 x-kubernetes-validations:
                 - message: clusterDNS is immutable
                   rule: self == oldSelf
+              clusterLimit:
+                description: Limit is the limits that apply for the server/worker
+                  nodes.
+                properties:
+                  serverLimit:
+                    additionalProperties:
+                      anyOf:
+                      - type: integer
+                      - type: string
+                      pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                      x-kubernetes-int-or-string: true
+                    description: ServerLimit is the limits (cpu/mem) that apply to
+                      the server nodes
+                    type: object
+                  workerLimit:
+                    additionalProperties:
+                      anyOf:
+                      - type: integer
+                      - type: string
+                      pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                      x-kubernetes-int-or-string: true
+                    description: WorkerLimit is the limits (cpu/mem) that apply to
+                      the agent nodes
+                    type: object
+                type: object
               expose:
                 description: |-
                   Expose contains options for exposing the apiserver inside/outside of the cluster. By default, this is only exposed as a
@@ -111,6 +136,12 @@ spec:
                 - loadbalancer
                 - nodePort
                 type: object
+              nodeSelector:
+                additionalProperties:
+                  type: string
+                description: NodeSelector is the node selector that will be applied
+                  to all server/agent pods
+                type: object
               persistence:
                 description: |-
                   Persistence contains options controlling how the etcd data of the virtual cluster is persisted. By default, no data

charts/k3k/crds/k3k.io_clustersets.yaml

@@ -0,0 +1,176 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.14.0
+  name: clustersets.k3k.io
+spec:
+  group: k3k.io
+  names:
+    kind: ClusterSet
+    listKind: ClusterSetList
+    plural: clustersets
+    singular: clusterset
+  scope: Namespaced
+  versions:
+  - name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: Spec is the spec of the ClusterSet
+            properties:
+              defaultLimits:
+                description: DefaultLimits are the limits used for servers/agents
+                  when a cluster in the set doesn't provide any
+                properties:
+                  serverLimit:
+                    additionalProperties:
+                      anyOf:
+                      - type: integer
+                      - type: string
+                      pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                      x-kubernetes-int-or-string: true
+                    description: ServerLimit is the limits (cpu/mem) that apply to
+                      the server nodes
+                    type: object
+                  workerLimit:
+                    additionalProperties:
+                      anyOf:
+                      - type: integer
+                      - type: string
+                      pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                      x-kubernetes-int-or-string: true
+                    description: WorkerLimit is the limits (cpu/mem) that apply to
+                      the agent nodes
+                    type: object
+                type: object
+              defaultNodeSelector:
+                additionalProperties:
+                  type: string
+                description: DefaultNodeSelector is the node selector that applies
+                  to all clusters (server + agent) in the set
+                type: object
+              maxLimits:
+                additionalProperties:
+                  anyOf:
+                  - type: integer
+                  - type: string
+                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                  x-kubernetes-int-or-string: true
+                description: MaxLimits are the limits that apply to all clusters (server
+                  + agent) in the set
+                type: object
+            type: object
+          status:
+            description: Status is the status of the ClusterSet
+            properties:
+              conditions:
+                description: Conditions are the invidual conditions for the cluster
+                  set
+                items:
+                  description: "Condition contains details for one aspect of the current
+                    state of this API Resource.\n---\nThis struct is intended for
+                    direct use as an array at the field path .status.conditions.  For
+                    example,\n\n\n\ttype FooStatus struct{\n\t    // Represents the
+                    observations of a foo's current state.\n\t    // Known .status.conditions.type
+                    are: \"Available\", \"Progressing\", and \"Degraded\"\n\t    //
+                    +patchMergeKey=type\n\t    // +patchStrategy=merge\n\t    // +listType=map\n\t
+                    \   // +listMapKey=type\n\t    Conditions []metav1.Condition `json:\"conditions,omitempty\"
+                    patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`\n\n\n\t
+                    \   // other fields\n\t}"
+                  properties:
+                    lastTransitionTime:
+                      description: |-
+                        lastTransitionTime is the last time the condition transitioned from one status to another.
+                        This should be when the underlying condition changed.  If that is not known, then using the time when the API field changed is acceptable.
+                      format: date-time
+                      type: string
+                    message:
+                      description: |-
+                        message is a human readable message indicating details about the transition.
+                        This may be an empty string.
+                      maxLength: 32768
+                      type: string
+                    observedGeneration:
+                      description: |-
+                        observedGeneration represents the .metadata.generation that the condition was set based upon.
+                        For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
+                        with respect to the current state of the instance.
+                      format: int64
+                      minimum: 0
+                      type: integer
+                    reason:
+                      description: |-
+                        reason contains a programmatic identifier indicating the reason for the condition's last transition.
+                        Producers of specific condition types may define expected values and meanings for this field,
+                        and whether the values are considered a guaranteed API.
+                        The value should be a CamelCase string.
+                        This field may not be empty.
+                      maxLength: 1024
+                      minLength: 1
+                      pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
+                      type: string
+                    status:
+                      description: status of the condition, one of True, False, Unknown.
+                      enum:
+                      - "True"
+                      - "False"
+                      - Unknown
+                      type: string
+                    type:
+                      description: |-
+                        type of condition in CamelCase or in foo.example.com/CamelCase.
+                        ---
+                        Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be
+                        useful (see .node.status.conditions), the ability to deconflict is important.
+                        The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)
+                      maxLength: 316
+                      pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
+                      type: string
+                  required:
+                  - lastTransitionTime
+                  - message
+                  - reason
+                  - status
+                  - type
+                  type: object
+                type: array
+              lastUpdateTime:
+                description: LastUpdate is the timestamp when the status was last
+                  updated
+                type: string
+              observedGeneration:
+                description: ObservedGeneration was the generation at the time the
+                  status was updated.
+                format: int64
+                type: integer
+              summary:
+                description: Sumamry is a summary of the status (error, ready)
+                type: string
+            type: object
+        required:
+        - spec
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}

charts/k3k/templates/deployment.yaml

@@ -15,6 +15,10 @@ spec:
       labels:
         {{- include "k3k.selectorLabels" . | nindent 8 }}
     spec:
+      volumes:
+        - name: webhook-serving
+          secret:
+            secretName: webhook-secret
       containers:
         - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
@@ -23,4 +27,11 @@ spec:
           - containerPort: 8080
             name: https
             protocol: TCP
-      serviceAccountName: {{ include "k3k.serviceAccountName" . }}
+          - containerPort: 9443
+            name: https-webhook
+            protocol: TCP
+          volumeMounts:
+            - name: webhook-serving
+              readOnly: true
+              mountPath: "/tmp/k8s-webhook-server/serving-certs"
+      serviceAccountName: {{ include "k3k.serviceAccountName" . }}

charts/k3k/templates/service.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata: 
+  name: k3k-webhook
+  labels:
+    {{- include "k3k.labels" . | nindent 4 }}
+  namespace: {{ .Values.namespace }}
+spec:
+  ports:
+    - port: 443
+      protocol: TCP
+      name: https-webhook
+      targetPort: 9443
+  selector:
+    {{- include "k3k.selectorLabels" . | nindent 6 }}

charts/k3k/templates/webhooks.yaml

@@ -0,0 +1,46 @@
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingWebhookConfiguration
+metadata:
+  name: "k3k.io.clusterset"
+webhooks:
+- name: "clusters.k3k.io"
+  rules:
+  - apiGroups:   ["k3k.io"]
+    apiVersions: ["v1alpha1"]
+    operations:  ["CREATE", "UPDATE"]
+    resources:   ["clusters"]
+    scope:       "Namespaced"
+  clientConfig:
+    service:
+      name: {{ include "k3k.fullname" . }}-webhook
+      namespace: {{ .Values.namespace }}
+      path: /validate-k3k-io-v1alpha1-cluster
+    caBundle: >
+      ReplaceMe
+  admissionReviewVersions: ["v1"]
+  sideEffects: None
+  timeoutSeconds: 10
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingWebhookConfiguration
+metadata:
+  name: "k3k.io.clusterset-default"
+webhooks:
+- name: "clusters.k3k.io"
+  rules:
+  - apiGroups:   ["k3k.io"]
+    apiVersions: ["v1alpha1"]
+    operations:  ["CREATE"]
+    resources:   ["clusters"]
+    scope:       "Namespaced"
+  clientConfig:
+    service:
+      name: {{ include "k3k.fullname" . }}-webhook
+      namespace: {{ .Values.namespace }}
+      path: /mutate-k3k-io-v1alpha1-cluster
+    caBundle: >
+      ReplaceMe
+  admissionReviewVersions: ["v1"]
+  sideEffects: None
+  timeoutSeconds: 10
+

charts/k3k/values.yaml

@@ -2,7 +2,7 @@ replicaCount: 1
 namespace: k3k-system
 
 image:
-  repository: rancher/k3k
+  repository: rancher/k3k 
   pullPolicy: Always
   # Overrides the image tag whose default is the chart appVersion.
   tag: "v0.2.1"

main.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
 	"github.com/rancher/k3k/pkg/controller/cluster"
+	"github.com/rancher/k3k/pkg/controller/clusterset"
 	"k8s.io/apimachinery/pkg/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/tools/clientcmd"
@@ -38,12 +39,25 @@ func main() {
 	mgr, err := ctrl.NewManager(restConfig, manager.Options{
 		Scheme: Scheme,
 	})
+
 	if err != nil {
 		klog.Fatalf("Failed to create new controller runtime manager: %v", err)
 	}
 
 	if err := cluster.Add(ctx, mgr); err != nil {
-		klog.Fatalf("Failed to add the new controller: %v", err)
+		klog.Fatalf("Failed to add the new cluster controller: %v", err)
+	}
+
+	if err := cluster.AddPodController(ctx, mgr); err != nil {
+		klog.Fatalf("Failed to add the new cluster controller: %v", err)
+	}
+	klog.Info("adding clusterset controller")
+	if err := clusterset.Add(ctx, mgr); err != nil {
+		klog.Fatalf("Failed to add the clusterset controller: %v", err)
+	}
+
+	if err := cluster.AddWebhookHandler(ctx, mgr); err != nil {
+		klog.Fatalf("failed to add a webhook for the cluster type: %v", err)
 	}
 
 	if err := cluster.AddPodController(ctx, mgr); err != nil {

pkg/apis/k3k.io/v1alpha1/register.go

@@ -21,7 +21,10 @@ func Resource(resource string) schema.GroupResource {
 func addKnownTypes(s *runtime.Scheme) error {
 	s.AddKnownTypes(SchemeGroupVersion,
 		&Cluster{},
-		&ClusterList{})
+		&ClusterList{},
+		&ClusterSet{},
+		&ClusterSetList{},
+	)
 	metav1.AddToGroupVersion(s, SchemeGroupVersion)
 	return nil
 }