From 0cc8edc5ecc12f51485b16d87fc4f94b2d3b2be5 Mon Sep 17 00:00:00 2001 From: Caleb Woodbine Date: Wed, 29 May 2024 17:16:21 +1200 Subject: [PATCH] feat: add defaults configurable runtimeClassName allow the setting of Pod RuntimeClassName via defaults --- config/core/configmaps/deployment.yaml | 17 +- pkg/deployment/config.go | 69 +++++ pkg/deployment/config_test.go | 240 ++++++++++++++++++ pkg/deployment/zz_generated.deepcopy.go | 30 +++ pkg/reconciler/revision/resources/deploy.go | 3 + .../revision/resources/deploy_test.go | 113 +++++++++ 6 files changed, 471 insertions(+), 1 deletion(-) diff --git a/config/core/configmaps/deployment.yaml b/config/core/configmaps/deployment.yaml index b8c70ab39da8..5813cb87fd8b 100644 --- a/config/core/configmaps/deployment.yaml +++ b/config/core/configmaps/deployment.yaml @@ -22,7 +22,7 @@ metadata: app.kubernetes.io/component: controller app.kubernetes.io/version: devel annotations: - knative.dev/example-checksum: "e2f637c6" + knative.dev/example-checksum: "720ddb97" data: # This is the Go import path for the binary that is containerized # and substituted here. @@ -108,3 +108,18 @@ data: # ` # This may be "none" or "prefer-spread-revision-over-nodes" (default) # default-affinity-type: "prefer-spread-revision-over-nodes" + + # runtime-class-name contains the selector for which runtimeClassName + # is selected to put in a revision. + # By default, it is not set by Knative. + # + # Example: + # runtime-class-name: | + # "": + # selector: + # use-default-runc: "yes" + # kata: {} + # gvisor: + # selector: + # use-gvisor: "please" + runtime-class-name: "" diff --git a/pkg/deployment/config.go b/pkg/deployment/config.go index 4bb78d33056d..9573eadbe000 100644 --- a/pkg/deployment/config.go +++ b/pkg/deployment/config.go @@ -19,12 +19,18 @@ package deployment import ( "errors" "fmt" + "strings" "time" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" + apimachineryvalidation "k8s.io/apimachinery/pkg/api/validation" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/sets" + "sigs.k8s.io/yaml" + cm "knative.dev/pkg/configmap" + "knative.dev/pkg/ptr" ) const ( @@ -70,6 +76,8 @@ const ( defaultAffinityTypeKey = "default-affinity-type" defaultAffinityTypeValue = PreferSpreadRevisionOverNodes + + RuntimeClassNameKey = "runtime-class-name" ) var ( @@ -116,10 +124,53 @@ func defaultConfig() *Config { return cfg } +func (d Config) PodRuntimeClassName(lbs map[string]string) *string { + runtimeClassName := "" + specificity := -1 + for k, v := range d.RuntimeClassNames { + if !v.Matches(lbs) || v.specificity() < specificity { + continue + } + if v.specificity() > specificity || strings.Compare(k, runtimeClassName) < 0 { + runtimeClassName = k + specificity = v.specificity() + } + } + if runtimeClassName == "" { + return nil + } + return ptr.String(runtimeClassName) +} + +type RuntimeClassNameLabelSelector struct { + Selector map[string]string `json:"selector,omitempty"` +} + +func (s *RuntimeClassNameLabelSelector) specificity() int { + if s.Selector == nil { + return 0 + } + return len(s.Selector) +} + +func (s *RuntimeClassNameLabelSelector) Matches(labels map[string]string) bool { + if s.Selector == nil { + return true + } + for label, expectedValue := range s.Selector { + value, ok := labels[label] + if !ok || expectedValue != value { + return false + } + } + return true +} + // NewConfigFromMap creates a DeploymentConfig from the supplied Map. func NewConfigFromMap(configMap map[string]string) (*Config, error) { nc := defaultConfig() + var runtimeClassNames string if err := cm.Parse(configMap, // Legacy keys for backwards compatibility cm.AsString(DeprecatedQueueSidecarImageKey, &nc.QueueSidecarImage), @@ -147,6 +198,8 @@ func NewConfigFromMap(configMap map[string]string) (*Config, error) { cm.AsStringSet(queueSidecarTokenAudiencesKey, &nc.QueueSidecarTokenAudiences), cm.AsString(queueSidecarRooCAKey, &nc.QueueSidecarRootCA), + + cm.AsString(RuntimeClassNameKey, &runtimeClassNames), ); err != nil { return nil, err } @@ -175,6 +228,19 @@ func NewConfigFromMap(configMap map[string]string) (*Config, error) { return nil, fmt.Errorf("unsupported %s value: %q", defaultAffinityTypeKey, affinity) } } + if err := yaml.Unmarshal([]byte(runtimeClassNames), &nc.RuntimeClassNames); err != nil { + return nil, fmt.Errorf("%v cannot be parsed, please check the format: %w", RuntimeClassNameKey, err) + } + for class, rcn := range nc.RuntimeClassNames { + if err := apimachineryvalidation.NameIsDNSSubdomain(class, false); len(err) > 0 { + return nil, fmt.Errorf("%v %v selector not valid DNSSubdomain: %w", RuntimeClassNameKey, class, err) + } + if len(rcn.Selector) > 0 { + if _, err := labels.ValidatedSelectorFromSet(rcn.Selector); err != nil { + return nil, fmt.Errorf("%v %v selector invalid: %w", RuntimeClassNameKey, class, err) + } + } + } return nc, nil } @@ -240,4 +306,7 @@ type Config struct { // DefaultAffinityType is a string that controls what affinity rules will be automatically // applied to the PodSpec of all Knative services. DefaultAffinityType AffinityType + + // RuntimeClassNames specifies which runtime the Pod will use + RuntimeClassNames map[string]RuntimeClassNameLabelSelector } diff --git a/pkg/deployment/config_test.go b/pkg/deployment/config_test.go index b4965126d5fd..301dd3782a91 100644 --- a/pkg/deployment/config_test.go +++ b/pkg/deployment/config_test.go @@ -17,16 +17,20 @@ limitations under the License. package deployment import ( + "strings" "testing" "time" "github.com/google/go-cmp/cmp" + "sigs.k8s.io/yaml" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" + "knative.dev/pkg/ptr" "knative.dev/pkg/system" "knative.dev/serving/test/conformance/api/shared" @@ -334,6 +338,124 @@ func TestControllerConfiguration(t *testing.T) { QueueSidecarTokenAudiences: sets.New("foo"), DefaultAffinityType: defaultAffinityTypeValue, }, + }, { + name: "runtime class name defaults to nothing", + wantErr: false, + data: map[string]string{ + QueueSidecarImageKey: defaultSidecarImage, + }, + wantConfig: &Config{ + DigestResolutionTimeout: digestResolutionTimeoutDefault, + ProgressDeadline: ProgressDeadlineDefault, + QueueSidecarCPURequest: &QueueSidecarCPURequestDefault, + QueueSidecarImage: defaultSidecarImage, + QueueSidecarTokenAudiences: sets.New(""), + RegistriesSkippingTagResolving: sets.New("kind.local", "ko.local", "dev.local"), + RuntimeClassNames: nil, + DefaultAffinityType: defaultAffinityTypeValue, + }, + }, { + name: "runtime class name with wildcard", + wantErr: false, + wantConfig: &Config{ + RuntimeClassNames: map[string]RuntimeClassNameLabelSelector{ + "gvisor": {}, + }, + DigestResolutionTimeout: digestResolutionTimeoutDefault, + ProgressDeadline: ProgressDeadlineDefault, + QueueSidecarCPURequest: &QueueSidecarCPURequestDefault, + QueueSidecarImage: defaultSidecarImage, + QueueSidecarTokenAudiences: sets.New(""), + RegistriesSkippingTagResolving: sets.New("kind.local", "ko.local", "dev.local"), + DefaultAffinityType: defaultAffinityTypeValue, + }, + data: map[string]string{ + RuntimeClassNameKey: "gvisor: {}", + QueueSidecarImageKey: defaultSidecarImage, + }, + }, { + name: "runtime class name with wildcard and label selectors", + wantErr: false, + wantConfig: &Config{ + RuntimeClassNames: map[string]RuntimeClassNameLabelSelector{ + "gvisor": {}, + "kata": { + Selector: map[string]string{ + "some": "value-here", + }, + }, + }, + DigestResolutionTimeout: digestResolutionTimeoutDefault, + ProgressDeadline: ProgressDeadlineDefault, + QueueSidecarCPURequest: &QueueSidecarCPURequestDefault, + QueueSidecarImage: defaultSidecarImage, + QueueSidecarTokenAudiences: sets.New(""), + RegistriesSkippingTagResolving: sets.New("kind.local", "ko.local", "dev.local"), + DefaultAffinityType: defaultAffinityTypeValue, + }, + data: map[string]string{ + RuntimeClassNameKey: `--- +gvisor: {} +kata: + selector: + some: value-here +`, + QueueSidecarImageKey: defaultSidecarImage, + }, + }, { + name: "runtime class name with bad label selectors", + wantErr: true, + data: map[string]string{ + QueueSidecarImageKey: defaultSidecarImage, + RuntimeClassNameKey: `--- +gvisor: {} +kata: + selector: + "-a": " a a " +`, + }, + }, { + name: "runtime class name with an unparsable format", + wantErr: true, + data: map[string]string{ + QueueSidecarImageKey: defaultSidecarImage, + RuntimeClassNameKey: ` ???; 231424 `, + }, + }, { + name: "invalid runtime class name", + wantErr: true, + data: map[string]string{ + QueueSidecarImageKey: defaultSidecarImage, + RuntimeClassNameKey: func() string { + badValues := []string{ + "", "A", "ABC", "aBc", "A1", "A-1", "1-A", + "-", "a-", "-a", "1-", "-1", + "_", "a_", "_a", "a_b", "1_", "_1", "1_2", + ".", "a.", ".a", "a..b", "1.", ".1", "1..2", + " ", "a ", " a", "a b", "1 ", " 1", "1 2", + "A.a", "aB.a", "ab.A", "A1.a", "a1.A", + "A.1", "aB.1", "A1.1", "1A.1", + "0.A", "01.A", "012.A", "1A.a", "1a.A", + "A.B.C.D.E", "AA.BB.CC.DD.EE", "a.B.c.d.e", "aa.bB.cc.dd.ee", + "a@b", "a,b", "a_b", "a;b", + "a:b", "a%b", "a?b", "a$b", + strings.Repeat("a", 254), + } + rcns := map[string]RuntimeClassNameLabelSelector{} + for _, v := range badValues { + rcns[v] = RuntimeClassNameLabelSelector{ + Selector: map[string]string{ + "unique": v, + }, + } + } + b, err := yaml.Marshal(rcns) + if err != nil { + panic(err) + } + return string(b) + }(), + }, }} for _, tt := range configTests { @@ -369,3 +491,121 @@ func quantity(val string) *resource.Quantity { r := resource.MustParse(val) return &r } + +func TestPodRuntimeClassName(t *testing.T) { + ts := []struct { + name string + serviceLabels map[string]string + runtimeClassNames map[string]RuntimeClassNameLabelSelector + want *string + }{{ + name: "empty", + serviceLabels: map[string]string{}, + runtimeClassNames: nil, + want: nil, + }, { + name: "wildcard set", + serviceLabels: map[string]string{}, + runtimeClassNames: map[string]RuntimeClassNameLabelSelector{ + "gvisor": {}, + }, + want: ptr.String("gvisor"), + }, { + name: "priority with multiple label selectors and one label set", + serviceLabels: map[string]string{ + "needs-two": "yes", + }, + runtimeClassNames: map[string]RuntimeClassNameLabelSelector{ + "one": {}, + "two": { + Selector: map[string]string{ + "needs-two": "yes", + }, + }, + "three": { + Selector: map[string]string{ + "needs-two": "yes", + "needs-three": "yes", + }, + }, + }, + want: ptr.String("two"), + }, { + name: "priority with multiple label selectors and two labels set", + serviceLabels: map[string]string{ + "needs-two": "yes", + "needs-three": "yes", + }, + runtimeClassNames: map[string]RuntimeClassNameLabelSelector{ + "one": {}, + "two": { + Selector: map[string]string{ + "needs-two": "yes", + }, + }, + "three": { + Selector: map[string]string{ + "needs-two": "yes", + "needs-three": "yes", + }, + }, + }, + want: ptr.String("three"), + }, { + name: "set via label", + serviceLabels: map[string]string{ + "very-cool": "indeed", + }, + runtimeClassNames: map[string]RuntimeClassNameLabelSelector{ + "gvisor": {}, + "kata": { + Selector: map[string]string{ + "very-cool": "indeed", + }, + }, + }, + want: ptr.String("kata"), + }, { + name: "no default only labels with set labels", + serviceLabels: map[string]string{ + "very-cool": "indeed", + }, + runtimeClassNames: map[string]RuntimeClassNameLabelSelector{ + "": {}, + "kata": { + Selector: map[string]string{ + "very-cool": "indeed", + }, + }, + }, + want: ptr.String("kata"), + }, { + name: "no default only labels with set no labels", + serviceLabels: map[string]string{}, + runtimeClassNames: map[string]RuntimeClassNameLabelSelector{ + "": {}, + "kata": { + Selector: map[string]string{ + "very-cool": "indeed", + }, + }, + }, + want: nil, + }} + + for _, tt := range ts { + tt := tt + t.Run(tt.name, func(t *testing.T) { + if tt.serviceLabels == nil { + tt.serviceLabels = map[string]string{} + } + defaults := defaultConfig() + defaults.RuntimeClassNames = tt.runtimeClassNames + got, want := defaults.PodRuntimeClassName(tt.serviceLabels), tt.want + + if !equality.Semantic.DeepEqual(got, want) { + t.Errorf("PodRuntimeClassName() = %v, wanted %v", got, want) + } + }) + } +} diff --git a/pkg/deployment/zz_generated.deepcopy.go b/pkg/deployment/zz_generated.deepcopy.go index f7362289a566..c3e6140e74ce 100644 --- a/pkg/deployment/zz_generated.deepcopy.go +++ b/pkg/deployment/zz_generated.deepcopy.go @@ -72,6 +72,13 @@ func (in *Config) DeepCopyInto(out *Config) { (*out)[key] = val } } + if in.RuntimeClassNames != nil { + in, out := &in.RuntimeClassNames, &out.RuntimeClassNames + *out = make(map[string]RuntimeClassNameLabelSelector, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } + } return } @@ -84,3 +91,26 @@ func (in *Config) DeepCopy() *Config { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RuntimeClassNameLabelSelector) DeepCopyInto(out *RuntimeClassNameLabelSelector) { + *out = *in + if in.Selector != nil { + in, out := &in.Selector, &out.Selector + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RuntimeClassNameLabelSelector. +func (in *RuntimeClassNameLabelSelector) DeepCopy() *RuntimeClassNameLabelSelector { + if in == nil { + return nil + } + out := new(RuntimeClassNameLabelSelector) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/reconciler/revision/resources/deploy.go b/pkg/reconciler/revision/resources/deploy.go index d17cfe830b54..f62c5514f2d3 100644 --- a/pkg/reconciler/revision/resources/deploy.go +++ b/pkg/reconciler/revision/resources/deploy.go @@ -210,6 +210,9 @@ func makePodSpec(rev *v1.Revision, cfg *config.Config) (*corev1.PodSpec, error) podSpec := BuildPodSpec(rev, append(BuildUserContainers(rev), *queueContainer), cfg) podSpec.Volumes = append(podSpec.Volumes, extraVolumes...) + if val := cfg.Deployment.PodRuntimeClassName(rev.ObjectMeta.Labels); podSpec.RuntimeClassName == nil { + podSpec.RuntimeClassName = val + } if cfg.Observability.EnableVarLogCollection { podSpec.Volumes = append(podSpec.Volumes, varLogVolume) diff --git a/pkg/reconciler/revision/resources/deploy_test.go b/pkg/reconciler/revision/resources/deploy_test.go index bf1038130499..2513e858ff1b 100644 --- a/pkg/reconciler/revision/resources/deploy_test.go +++ b/pkg/reconciler/revision/resources/deploy_test.go @@ -407,6 +407,12 @@ func withPrependedVolumeMounts(volumeMounts ...corev1.VolumeMount) containerOpti } } +func withRuntimeClass(name string) podSpecOption { + return func(ps *corev1.PodSpec) { + ps.RuntimeClassName = ptr.String(name) + } +} + func podSpec(containers []corev1.Container, opts ...podSpecOption) *corev1.PodSpec { podSpec := defaultPodSpec.DeepCopy() podSpec.Containers = containers @@ -1526,6 +1532,113 @@ func TestMakePodSpec(t *testing.T) { } }, ), + }, { + name: "with runtime-class-name set", + dc: deployment.Config{ + RuntimeClassNames: map[string]deployment.RuntimeClassNameLabelSelector{ + "gvisor": {}, + }, + }, + rev: revision("bar", "foo", + withContainers([]corev1.Container{{ + Name: servingContainerName, + Image: "busybox", + Ports: buildContainerPorts(v1.DefaultUserPort), + ReadinessProbe: withHTTPReadinessProbe(v1.DefaultUserPort), + }}), + ), + want: podSpec([]corev1.Container{ + servingContainer(func(container *corev1.Container) { + container.Image = "busybox" + }), + queueContainer( + withEnvVar("SERVING_READINESS_PROBE", `{"httpGet":{"path":"/","port":8080,"host":"127.0.0.1","scheme":"HTTP"}}`), + ), + }, withRuntimeClass("gvisor")), + }, { + name: "with runtime-class-name set requiring selector and no label set in revision", + dc: deployment.Config{ + RuntimeClassNames: map[string]deployment.RuntimeClassNameLabelSelector{ + "gvisor": { + Selector: map[string]string{ + "this-one": "specifically", + }, + }, + }, + }, + rev: revision("bar", "foo", + withContainers([]corev1.Container{{ + Name: servingContainerName, + Image: "busybox", + Ports: buildContainerPorts(v1.DefaultUserPort), + ReadinessProbe: withHTTPReadinessProbe(v1.DefaultUserPort), + }}), + ), + want: podSpec([]corev1.Container{ + servingContainer(func(container *corev1.Container) { + container.Image = "busybox" + }), + queueContainer( + withEnvVar("SERVING_READINESS_PROBE", `{"httpGet":{"path":"/","port":8080,"host":"127.0.0.1","scheme":"HTTP"}}`), + ), + }), + }, { + name: "with runtime-class-name set requiring selector and label set in revision", + dc: deployment.Config{ + RuntimeClassNames: map[string]deployment.RuntimeClassNameLabelSelector{ + "gvisor": { + Selector: map[string]string{ + "this-one": "specifically", + }, + }, + }, + }, + rev: revision("bar", "foo", + WithRevisionLabel("this-one", "specifically"), + withContainers([]corev1.Container{{ + Name: servingContainerName, + Image: "busybox", + Ports: buildContainerPorts(v1.DefaultUserPort), + ReadinessProbe: withHTTPReadinessProbe(v1.DefaultUserPort), + }}), + ), + want: podSpec([]corev1.Container{ + servingContainer(func(container *corev1.Container) { + container.Image = "busybox" + }), + queueContainer( + withEnvVar("SERVING_READINESS_PROBE", `{"httpGet":{"path":"/","port":8080,"host":"127.0.0.1","scheme":"HTTP"}}`), + ), + }, withRuntimeClass("gvisor")), + }, { + name: "with multiple runtime-class-name set and label selector for one", + dc: deployment.Config{ + RuntimeClassNames: map[string]deployment.RuntimeClassNameLabelSelector{ + "gvisor": {}, + "kata": { + Selector: map[string]string{ + "specific": "this-one", + }, + }, + }, + }, + rev: revision("bar", "foo", + WithRevisionLabel("specific", "this-one"), + withContainers([]corev1.Container{{ + Name: servingContainerName, + Image: "busybox", + Ports: buildContainerPorts(v1.DefaultUserPort), + ReadinessProbe: withHTTPReadinessProbe(v1.DefaultUserPort), + }}), + ), + want: podSpec([]corev1.Container{ + servingContainer(func(container *corev1.Container) { + container.Image = "busybox" + }), + queueContainer( + withEnvVar("SERVING_READINESS_PROBE", `{"httpGet":{"path":"/","port":8080,"host":"127.0.0.1","scheme":"HTTP"}}`), + ), + }, withRuntimeClass("kata")), }} for _, test := range tests {