diff --git a/docs/tasks.md b/docs/tasks.md index a461f31a649..fea56f25a11 100644 --- a/docs/tasks.md +++ b/docs/tasks.md @@ -129,6 +129,12 @@ or container images that you define: the configuration file. - Each container image runs until completion or until the first failure is detected. +- The CPU/memory resource request fields will be set to zero if the container + image does not have the largest CPU/memory resource request out of all + container images in the Task. This ensures that the Pod that executes the Task + will only request the resources necessary to execute any single container + image in the Task, rather than requesting the sum of all of the container + image's resource requests. ### Inputs diff --git a/pkg/reconciler/v1alpha1/taskrun/resources/pod.go b/pkg/reconciler/v1alpha1/taskrun/resources/pod.go index f6ca4a9cecf..3619b139088 100644 --- a/pkg/reconciler/v1alpha1/taskrun/resources/pod.go +++ b/pkg/reconciler/v1alpha1/taskrun/resources/pod.go @@ -28,6 +28,7 @@ import ( "path/filepath" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/kubernetes" @@ -73,6 +74,8 @@ var ( VolumeSource: emptyVolumeSource, }} + zeroQty = resource.MustParse("0") + // Random byte reader used for pod name generation. // var for testing. randReader = rand.Reader @@ -174,6 +177,9 @@ func MakePod(taskRun *v1alpha1.TaskRun, taskSpec v1alpha1.TaskSpec, kubeclient k initContainers := []corev1.Container{*cred} podContainers := []corev1.Container{} + maxCpuIdx := findMaxResourceRequest(taskSpec.Steps, corev1.ResourceCPU) + maxMemIdx := findMaxResourceRequest(taskSpec.Steps, corev1.ResourceMemory) + for i, step := range taskSpec.Steps { step.Env = append(implicitEnvVars, step.Env...) // TODO(mattmoor): Check that volumeMounts match volumes. @@ -202,6 +208,7 @@ func MakePod(taskRun *v1alpha1.TaskRun, taskSpec v1alpha1.TaskSpec, kubeclient k if step.Name == names.SimpleNameGenerator.RestrictLength(fmt.Sprintf("%v%v", containerPrefix, entrypoint.InitContainerName)) { initContainers = append(initContainers, step) } else { + zeroNonMaxResourceRequests(&step, i == maxCpuIdx, i == maxMemIdx) podContainers = append(podContainers, step) } } @@ -261,3 +268,36 @@ func makeLabels(s *v1alpha1.TaskRun) map[string]string { labels[pipeline.GroupName+pipeline.TaskRunLabelKey] = s.Name return labels } + +// zeroNonMaxResourceRequests zeroes out the container's cpu or memory resource +// requests if the container does not have the largest request out of all +// containers in the pod. This is done because Tekton overwrites each +// container's entrypoint to make containers effectively execute one at a time, +// so we want pods to only request the maximum resources needed at any single point in time. +// If no contianer has an explicit resource request, all requests are set to 0. +func zeroNonMaxResourceRequests(container *corev1.Container, isMaxCpu, isMaxMem bool) { + if container.Resources.Requests == nil { + container.Resources.Requests = corev1.ResourceList{} + } + if !isMaxCpu { + container.Resources.Requests[corev1.ResourceCPU] = zeroQty + } + if !isMaxMem { + container.Resources.Requests[corev1.ResourceMemory] = zeroQty + } +} + +// findMaxResourceRequest returns the index of the container with the maximum +// request for the given resource from among the given set of containers. +func findMaxResourceRequest(containers []corev1.Container, resourceName corev1.ResourceName) int { + maxIdx := -1 + maxReq := zeroQty + for i, c := range containers { + req, exists := c.Resources.Requests[resourceName] + if exists && req.Cmp(maxReq) > 0 { + maxIdx = i + maxReq = req + } + } + return maxIdx +} diff --git a/pkg/reconciler/v1alpha1/taskrun/resources/pod_test.go b/pkg/reconciler/v1alpha1/taskrun/resources/pod_test.go index 3c67598d40f..fbdfde64e5d 100644 --- a/pkg/reconciler/v1alpha1/taskrun/resources/pod_test.go +++ b/pkg/reconciler/v1alpha1/taskrun/resources/pod_test.go @@ -22,7 +22,6 @@ import ( "testing" "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -33,8 +32,10 @@ import ( ) var ( - ignorePrivateResourceFields = cmpopts.IgnoreUnexported(resource.Quantity{}) - nopContainer = corev1.Container{ + resourceQuantityCmp = cmp.Comparer(func(x, y resource.Quantity) bool { + return x.Cmp(y) == 0 + }) + nopContainer = corev1.Container{ Name: "nop", Image: *nopImage, Command: []string{"/ko-app/nop"}, @@ -104,6 +105,12 @@ func TestMakePod(t *testing.T) { Env: implicitEnvVars, VolumeMounts: implicitVolumeMounts, WorkingDir: workspaceDir, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("0"), + corev1.ResourceMemory: resource.MustParse("0"), + }, + }, }, nopContainer, }, @@ -143,6 +150,12 @@ func TestMakePod(t *testing.T) { Env: implicitEnvVars, VolumeMounts: implicitVolumeMounts, WorkingDir: workspaceDir, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("0"), + corev1.ResourceMemory: resource.MustParse("0"), + }, + }, }, nopContainer, }, @@ -176,6 +189,12 @@ func TestMakePod(t *testing.T) { Env: implicitEnvVars, VolumeMounts: implicitVolumeMounts, WorkingDir: workspaceDir, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("0"), + corev1.ResourceMemory: resource.MustParse("0"), + }, + }, }, nopContainer, }, @@ -209,6 +228,12 @@ func TestMakePod(t *testing.T) { Env: implicitEnvVars, VolumeMounts: implicitVolumeMounts, WorkingDir: workspaceDir, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("0"), + corev1.ResourceMemory: resource.MustParse("0"), + }, + }, }, nopContainer, }, @@ -257,7 +282,7 @@ func TestMakePod(t *testing.T) { t.Errorf("Pod name got %q, want %q", got.Name, wantName) } - if d := cmp.Diff(&got.Spec, c.want, ignorePrivateResourceFields); d != "" { + if d := cmp.Diff(&got.Spec, c.want, resourceQuantityCmp); d != "" { t.Errorf("Diff spec:\n%s", d) } diff --git a/pkg/reconciler/v1alpha1/taskrun/taskrun_test.go b/pkg/reconciler/v1alpha1/taskrun/taskrun_test.go index 320e774a908..73cdd5f0689 100644 --- a/pkg/reconciler/v1alpha1/taskrun/taskrun_test.go +++ b/pkg/reconciler/v1alpha1/taskrun/taskrun_test.go @@ -37,6 +37,7 @@ import ( "go.uber.org/zap" "go.uber.org/zap/zaptest/observer" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" fakekubeclientset "k8s.io/client-go/kubernetes/fake" @@ -62,6 +63,9 @@ var ( }, cmp.Comparer(func(name1, name2 string) bool { return name1[:len(name1)-6] == name2[:len(name2)-6] })) + resourceQuantityCmp = cmp.Comparer(func(x, y resource.Quantity) bool { + return x.Cmp(y) == 0 + }) simpleStep = tb.Step("simple-step", "foo", tb.Command("/mycmd")) simpleTask = tb.Task("test-task", "foo", tb.TaskSpec(simpleStep)) @@ -272,11 +276,44 @@ func TestReconcile(t *testing.T) { ), ) + taskRunWithResourceRequests := tb.TaskRun("test-taskrun-with-resource-requests", "foo", + tb.TaskRunSpec( + tb.TaskRunTaskSpec( + tb.Step("step1", "foo", + tb.Command("/mycmd"), + tb.Resources( + tb.Limits( + tb.CPU("8"), + tb.Memory("10Gi"), + ), + tb.Requests( + tb.CPU("4"), + tb.Memory("3Gi"), + ), + ), + ), + tb.Step("step2", "foo", + tb.Command("/mycmd"), + tb.Resources( + tb.Limits(tb.Memory("5Gi")), + tb.Requests( + tb.CPU("2"), + tb.Memory("5Gi"), + ), + ), + ), + tb.Step("step3", "foo", + tb.Command("/mycmd"), + ), + ), + ), + ) + taskruns := []*v1alpha1.TaskRun{ taskRunSuccess, taskRunWithSaSuccess, taskRunTemplating, taskRunInputOutput, taskRunWithTaskSpec, taskRunWithClusterTask, taskRunWithResourceSpecAndTaskSpec, - taskRunWithLabels, + taskRunWithLabels, taskRunWithResourceRequests, } d := test.Data{ @@ -311,6 +348,10 @@ func TestReconcile(t *testing.T) { tb.VolumeMount("tools", "/builder/tools"), tb.VolumeMount("workspace", workspaceDir), tb.VolumeMount("home", "/builder/home"), + tb.Resources(tb.Requests( + tb.CPU("0"), + tb.Memory("0"), + )), ), tb.PodContainer("nop", "override-with-nop:latest", tb.Command("/ko-app/nop")), ), @@ -338,6 +379,10 @@ func TestReconcile(t *testing.T) { tb.VolumeMount("tools", "/builder/tools"), tb.VolumeMount("workspace", workspaceDir), tb.VolumeMount("home", "/builder/home"), + tb.Resources(tb.Requests( + tb.CPU("0"), + tb.Memory("0"), + )), ), tb.PodContainer("nop", "override-with-nop:latest", tb.Command("/ko-app/nop")), ), @@ -371,6 +416,10 @@ func TestReconcile(t *testing.T) { tb.VolumeMount("tools", "/builder/tools"), tb.VolumeMount("workspace", workspaceDir), tb.VolumeMount("home", "/builder/home"), + tb.Resources(tb.Requests( + tb.CPU("0"), + tb.Memory("0"), + )), ), tb.PodContainer("build-step-mycontainer", "myimage", tb.Command(entrypointLocation), @@ -382,6 +431,10 @@ func TestReconcile(t *testing.T) { tb.VolumeMount("tools", "/builder/tools"), tb.VolumeMount("workspace", workspaceDir), tb.VolumeMount("home", "/builder/home"), + tb.Resources(tb.Requests( + tb.CPU("0"), + tb.Memory("0"), + )), ), tb.PodContainer("build-step-myothercontainer", "myotherimage", tb.Command(entrypointLocation), @@ -392,6 +445,10 @@ func TestReconcile(t *testing.T) { tb.VolumeMount("tools", "/builder/tools"), tb.VolumeMount("workspace", workspaceDir), tb.VolumeMount("home", "/builder/home"), + tb.Resources(tb.Requests( + tb.CPU("0"), + tb.Memory("0"), + )), ), tb.PodContainer("nop", "override-with-nop:latest", tb.Command("/ko-app/nop")), ), @@ -427,6 +484,10 @@ func TestReconcile(t *testing.T) { tb.VolumeMount("tools", "/builder/tools"), tb.VolumeMount("workspace", workspaceDir), tb.VolumeMount("home", "/builder/home"), + tb.Resources(tb.Requests( + tb.CPU("0"), + tb.Memory("0"), + )), ), tb.PodContainer("build-step-source-copy-another-git-resource-mssqb", "override-with-bash-noop:latest", tb.Command(entrypointLocation), @@ -438,6 +499,10 @@ func TestReconcile(t *testing.T) { tb.VolumeMount("tools", "/builder/tools"), tb.VolumeMount("workspace", workspaceDir), tb.VolumeMount("home", "/builder/home"), + tb.Resources(tb.Requests( + tb.CPU("0"), + tb.Memory("0"), + )), ), tb.PodContainer("build-step-create-dir-git-resource-mz4c7", "override-with-bash-noop:latest", tb.Command(entrypointLocation), @@ -448,6 +513,10 @@ func TestReconcile(t *testing.T) { tb.VolumeMount("tools", "/builder/tools"), tb.VolumeMount("workspace", workspaceDir), tb.VolumeMount("home", "/builder/home"), + tb.Resources(tb.Requests( + tb.CPU("0"), + tb.Memory("0"), + )), ), tb.PodContainer("build-step-source-copy-git-resource-9l9zj", "override-with-bash-noop:latest", tb.Command(entrypointLocation), @@ -459,6 +528,10 @@ func TestReconcile(t *testing.T) { tb.VolumeMount("tools", "/builder/tools"), tb.VolumeMount("workspace", workspaceDir), tb.VolumeMount("home", "/builder/home"), + tb.Resources(tb.Requests( + tb.CPU("0"), + tb.Memory("0"), + )), ), tb.PodContainer("build-step-simple-step", "foo", tb.Command(entrypointLocation), @@ -468,6 +541,10 @@ func TestReconcile(t *testing.T) { tb.VolumeMount("tools", "/builder/tools"), tb.VolumeMount("workspace", workspaceDir), tb.VolumeMount("home", "/builder/home"), + tb.Resources(tb.Requests( + tb.CPU("0"), + tb.Memory("0"), + )), ), tb.PodContainer("build-step-source-mkdir-git-resource-6nl7g", "override-with-bash-noop:latest", tb.Command(entrypointLocation), @@ -479,6 +556,10 @@ func TestReconcile(t *testing.T) { tb.VolumeMount("tools", "/builder/tools"), tb.VolumeMount("workspace", workspaceDir), tb.VolumeMount("home", "/builder/home"), + tb.Resources(tb.Requests( + tb.CPU("0"), + tb.Memory("0"), + )), ), tb.PodContainer("build-step-source-copy-git-resource-j2tds", "override-with-bash-noop:latest", tb.Command(entrypointLocation), @@ -490,6 +571,10 @@ func TestReconcile(t *testing.T) { tb.VolumeMount("tools", "/builder/tools"), tb.VolumeMount("workspace", workspaceDir), tb.VolumeMount("home", "/builder/home"), + tb.Resources(tb.Requests( + tb.CPU("0"), + tb.Memory("0"), + )), ), tb.PodContainer("nop", "override-with-nop:latest", tb.Command("/ko-app/nop")), ), @@ -515,6 +600,10 @@ func TestReconcile(t *testing.T) { tb.VolumeMount("tools", "/builder/tools"), tb.VolumeMount("workspace", workspaceDir), tb.VolumeMount("home", "/builder/home"), + tb.Resources(tb.Requests( + tb.CPU("0"), + tb.Memory("0"), + )), ), placeToolsInitContainer, tb.PodContainer("build-step-mycontainer", "myimage", @@ -526,6 +615,10 @@ func TestReconcile(t *testing.T) { tb.VolumeMount("tools", "/builder/tools"), tb.VolumeMount("workspace", workspaceDir), tb.VolumeMount("home", "/builder/home"), + tb.Resources(tb.Requests( + tb.CPU("0"), + tb.Memory("0"), + )), ), tb.PodContainer("nop", "override-with-nop:latest", tb.Command("/ko-app/nop")), ), @@ -552,6 +645,10 @@ func TestReconcile(t *testing.T) { tb.VolumeMount("tools", "/builder/tools"), tb.VolumeMount("workspace", workspaceDir), tb.VolumeMount("home", "/builder/home"), + tb.Resources(tb.Requests( + tb.CPU("0"), + tb.Memory("0"), + )), ), tb.PodContainer("nop", "override-with-nop:latest", tb.Command("/ko-app/nop")), ), @@ -579,6 +676,10 @@ func TestReconcile(t *testing.T) { tb.VolumeMount("tools", "/builder/tools"), tb.VolumeMount("workspace", workspaceDir), tb.VolumeMount("home", "/builder/home"), + tb.Resources(tb.Requests( + tb.CPU("0"), + tb.Memory("0"), + )), ), tb.PodContainer("build-step-mystep", "ubuntu", tb.Command(entrypointLocation), @@ -588,6 +689,10 @@ func TestReconcile(t *testing.T) { tb.VolumeMount("tools", "/builder/tools"), tb.VolumeMount("workspace", workspaceDir), tb.VolumeMount("home", "/builder/home"), + tb.Resources(tb.Requests( + tb.CPU("0"), + tb.Memory("0"), + )), ), tb.PodContainer("nop", "override-with-nop:latest", tb.Command("/ko-app/nop")), ), @@ -615,6 +720,74 @@ func TestReconcile(t *testing.T) { tb.VolumeMount("tools", "/builder/tools"), tb.VolumeMount("workspace", workspaceDir), tb.VolumeMount("home", "/builder/home"), + tb.Resources(tb.Requests( + tb.CPU("0"), + tb.Memory("0"), + )), + ), + tb.PodContainer("nop", "override-with-nop:latest", tb.Command("/ko-app/nop")), + ), + ), + }, { + name: "taskrun-with-resource-requests", + taskRun: taskRunWithResourceRequests, + wantPod: tb.Pod("test-taskrun-with-resource-requests-pod-123456", "foo", + tb.PodAnnotation("sidecar.istio.io/inject", "false"), + tb.PodLabel(taskRunNameLabelKey, "test-taskrun-with-resource-requests"), + tb.PodOwnerReference("TaskRun", "test-taskrun-with-resource-requests", + tb.OwnerReferenceAPIVersion(currentApiVersion)), + tb.PodSpec( + tb.PodVolumes(toolsVolume, workspaceVolume, homeVolume), + tb.PodRestartPolicy(corev1.RestartPolicyNever), + getCredentialsInitContainer("9l9zj"), + placeToolsInitContainer, + tb.PodContainer("build-step-step1", "foo", + tb.Command(entrypointLocation), + tb.Args("-wait_file", "", "-post_file", "/builder/tools/0", "-entrypoint", "/mycmd", "--"), + tb.WorkingDir(workspaceDir), + tb.EnvVar("HOME", "/builder/home"), + tb.VolumeMount("tools", "/builder/tools"), + tb.VolumeMount("workspace", workspaceDir), + tb.VolumeMount("home", "/builder/home"), + tb.Resources( + tb.Limits( + tb.CPU("8"), + tb.Memory("10Gi"), + ), + tb.Requests( + tb.CPU("4"), + tb.Memory("0"), + ), + ), + ), + tb.PodContainer("build-step-step2", "foo", + tb.Command(entrypointLocation), + tb.Args("-wait_file", "/builder/tools/0", "-post_file", "/builder/tools/1", "-entrypoint", "/mycmd", "--"), + tb.WorkingDir(workspaceDir), + tb.EnvVar("HOME", "/builder/home"), + tb.VolumeMount("tools", "/builder/tools"), + tb.VolumeMount("workspace", workspaceDir), + tb.VolumeMount("home", "/builder/home"), + tb.Resources( + tb.Limits(tb.Memory("5Gi"),), + tb.Requests( + tb.CPU("0"), + tb.Memory("5Gi"), + ), + ), + ), + tb.PodContainer("build-step-step3", "foo", + tb.Command(entrypointLocation), + tb.Args("-wait_file", "/builder/tools/1", "-post_file", "/builder/tools/2", "-entrypoint", "/mycmd", "--"), + tb.WorkingDir(workspaceDir), + tb.EnvVar("HOME", "/builder/home"), + tb.VolumeMount("tools", "/builder/tools"), + tb.VolumeMount("workspace", workspaceDir), + tb.VolumeMount("home", "/builder/home"), + tb.Resources(tb.Requests( + tb.CPU("0"), + tb.Memory("0"), + )), ), tb.PodContainer("nop", "override-with-nop:latest", tb.Command("/ko-app/nop")), ), @@ -676,7 +849,7 @@ func TestReconcile(t *testing.T) { t.Errorf("Pod metadata doesn't match, diff: %s", d) } - if d := cmp.Diff(pod.Spec, tc.wantPod.Spec); d != "" { + if d := cmp.Diff(pod.Spec, tc.wantPod.Spec, resourceQuantityCmp); d != "" { t.Errorf("Pod spec doesn't match, diff: %s", d) } if len(clients.Kube.Actions()) == 0 { diff --git a/test/builder/container.go b/test/builder/container.go index 242cd3d9f0c..675523b864a 100644 --- a/test/builder/container.go +++ b/test/builder/container.go @@ -15,6 +15,7 @@ package builder import ( corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" ) // ContainerOp is an operation which modifies a Container struct. @@ -23,6 +24,12 @@ type ContainerOp func(*corev1.Container) // VolumeMountOp is an operation which modifies a VolumeMount struct. type VolumeMountOp func(*corev1.VolumeMount) +// ResourceRequirementsOp is an operation which modifies a ResourceRequirements struct. +type ResourceRequirementsOp func(*corev1.ResourceRequirements) + +// ResourceListOp is an operation which modifies a ResourceList struct. +type ResourceListOp func(corev1.ResourceList) + // Command sets the command to the Container (step in this case). func Command(args ...string) ContainerOp { return func(container *corev1.Container) { @@ -67,3 +74,45 @@ func VolumeMount(name, mountPath string, ops ...VolumeMountOp) ContainerOp { c.VolumeMounts = append(c.VolumeMounts, *mount) } } + +func Resources(ops ...ResourceRequirementsOp) ContainerOp { + return func(c *corev1.Container) { + rr := &corev1.ResourceRequirements{} + for _, op := range ops { + op(rr) + } + c.Resources = *rr + } +} + +func Limits(ops ...ResourceListOp) ResourceRequirementsOp { + return func(rr *corev1.ResourceRequirements) { + limits := corev1.ResourceList{} + for _, op := range ops { + op(limits) + } + rr.Limits = limits + } +} + +func Requests(ops ...ResourceListOp) ResourceRequirementsOp { + return func(rr *corev1.ResourceRequirements) { + requests := corev1.ResourceList{} + for _, op := range ops { + op(requests) + } + rr.Requests = requests + } +} + +func CPU(val string) ResourceListOp { + return func(r corev1.ResourceList) { + r[corev1.ResourceCPU] = resource.MustParse(val) + } +} + +func Memory(val string) ResourceListOp { + return func(r corev1.ResourceList) { + r[corev1.ResourceMemory] = resource.MustParse(val) + } +} diff --git a/test/builder/pod_test.go b/test/builder/pod_test.go index 3e7a2c57f8c..21ed4d40975 100644 --- a/test/builder/pod_test.go +++ b/test/builder/pod_test.go @@ -19,11 +19,15 @@ import ( "github.com/google/go-cmp/cmp" tb "github.com/tektoncd/pipeline/test/builder" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func TestPod(t *testing.T) { trueB := true + resourceQuantityCmp := cmp.Comparer(func(x, y resource.Quantity) bool { + return x.Cmp(y) == 0 + }) volume := corev1.Volume{ Name: "tools-volume", VolumeSource: corev1.VolumeSource{}, @@ -43,6 +47,13 @@ func TestPod(t *testing.T) { tb.WorkingDir("/workspace"), tb.EnvVar("HOME", "/builder/home"), tb.VolumeMount("tools-volume", "/tools"), + tb.Resources( + tb.Limits(tb.Memory("1.5Gi")), + tb.Requests( + tb.CPU("100m"), + tb.Memory("1Gi"), + ), + ), ), tb.PodVolumes(volume), ), @@ -86,11 +97,20 @@ func TestPod(t *testing.T) { Name: "tools-volume", MountPath: "/tools", }}, + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("1.5Gi"), + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + corev1.ResourceMemory: resource.MustParse("1Gi"), + }, + }, }}, Volumes: []corev1.Volume{volume}, }, } - if d := cmp.Diff(expectedPod, pod); d != "" { + if d := cmp.Diff(expectedPod, pod, resourceQuantityCmp); d != "" { t.Fatalf("Pod diff -want, +got: %v", d) } }