diff --git a/docs/compute-resources.md b/docs/compute-resources.md index d4446032403..7f7fbb3b6b9 100644 --- a/docs/compute-resources.md +++ b/docs/compute-resources.md @@ -45,7 +45,7 @@ Therefore, the pod will have no effective CPU limit. ## Task-level Compute Resources Configuration **([alpha only](https://github.com/tektoncd/pipeline/blob/main/docs/install.md#alpha-features))** -(This feature is under development and not functional yet. Stay tuned!) +(This alpha feature is implemented) Tekton allows users to specify resource requirements of [`Steps`](./tasks.md#defining-steps), which run sequentially. However, the pod's effective resource requirements are still the @@ -63,7 +63,9 @@ Each of these details is explained in more depth below. Some points to note: - Task-level resource requests and limits do not apply to sidecars which can be configured separately. -- Users may not configure the Task-level and Step-level resource requirements (requests/limits) simultaneously. +- If only limits are configured in task-level, it will be applied as the task-level requests. +- Resource requirements configured in `Step` or `StepTemplate` of the referenced `Task` will be overridden by the task-level requirements. +- `TaskRun` configured with both `StepOverrides` and task-level requirements will be rejected. ### Configure Task-level Compute Resources @@ -84,7 +86,7 @@ spec: cpu: 2 ``` -The following TaskRun will be rejected, because it configures both step-level and task-level compute resource requirements: +The following TaskRun will be rejected, because it configures both stepOverrides and task-level compute resource requirements: ```yaml kind: TaskRun diff --git a/pkg/internal/computeresources/tasklevel/tasklevel.go b/pkg/internal/computeresources/tasklevel/tasklevel.go new file mode 100644 index 00000000000..447affa6a5a --- /dev/null +++ b/pkg/internal/computeresources/tasklevel/tasklevel.go @@ -0,0 +1,57 @@ +/* +Copyright 2020 The Tekton Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasklevel + +import ( + "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1beta1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" +) + +// ApplyTaskLevelComputeResources applies the task-level compute resource requirements to each Step. +func ApplyTaskLevelComputeResources(steps []v1beta1.Step, computeResources *corev1.ResourceRequirements) { + if computeResources == nil { + return + } + if computeResources.Requests == nil && computeResources.Limits == nil { + return + } + averageRequests := computeAverageRequests(computeResources.Requests, len(steps)) + averageLimits := computeAverageRequests(computeResources.Limits, len(steps)) + for i := range steps { + // if no requests are specified in step or task level, the limits are used to avoid + // unnecessary higher requests by Kubernetes default behavior. + if steps[i].Resources.Requests == nil && computeResources.Requests == nil { + steps[i].Resources.Requests = averageLimits + } else { + steps[i].Resources.Requests = averageRequests + } + steps[i].Resources.Limits = computeResources.Limits + } +} + +// computeAverageRequests computes the average of the requests of all the steps. +func computeAverageRequests(requests corev1.ResourceList, steps int) corev1.ResourceList { + if len(requests) == 0 || steps == 0 { + return nil + } + averageRequests := corev1.ResourceList{} + for k, v := range requests { + averageRequests[k] = *resource.NewMilliQuantity(v.MilliValue()/int64(steps), requests[k].Format) + } + return averageRequests +} diff --git a/pkg/internal/computeresources/tasklevel/tasklevel_test.go b/pkg/internal/computeresources/tasklevel/tasklevel_test.go new file mode 100644 index 00000000000..00114476681 --- /dev/null +++ b/pkg/internal/computeresources/tasklevel/tasklevel_test.go @@ -0,0 +1,206 @@ +/* +Copyright 2019 The Tekton Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasklevel_test + +import ( + "fmt" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1beta1" + "github.com/tektoncd/pipeline/pkg/internal/computeresources/tasklevel" + "github.com/tektoncd/pipeline/test/diff" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" +) + +func TestApplyTaskLevelResourceRequirements(t *testing.T) { + testcases := []struct { + desc string + Steps []v1beta1.Step + ComputeResources corev1.ResourceRequirements + expectedComputeResources []corev1.ResourceRequirements + }{{ + desc: "only with requests", + Steps: []v1beta1.Step{{ + Name: "1st-step", + Image: "image", + Command: []string{"cmd"}, + }, { + Name: "2nd-step", + Image: "image", + Command: []string{"cmd"}, + }}, + ComputeResources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("2")}, + }, + expectedComputeResources: []corev1.ResourceRequirements{{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1")}, + }, { + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1")}, + }}, + }, { + desc: "only with limits", + Steps: []v1beta1.Step{{ + Name: "1st-step", + Image: "image", + Command: []string{"cmd"}, + }, { + Name: "2nd-step", + Image: "image", + Command: []string{"cmd"}, + }}, + ComputeResources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("500m")}, + }, + expectedComputeResources: []corev1.ResourceRequirements{{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("250m")}, + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("500m")}, + }, { + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("250m")}, + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("500m")}, + }}, + }, { + desc: "both with requests and limits", + Steps: []v1beta1.Step{{ + Name: "1st-step", + Image: "image", + Command: []string{"cmd"}, + }, { + Name: "2nd-step", + Image: "image", + Command: []string{"cmd"}, + }}, + ComputeResources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1")}, + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("2")}, + }, + expectedComputeResources: []corev1.ResourceRequirements{{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("500m")}, + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("2")}, + }, { + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("500m")}, + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("2")}, + }}, + }, { + desc: "steps with compute resources are overridden by task-level compute resources", + Steps: []v1beta1.Step{{ + Name: "1st-step", + Image: "image", + Command: []string{"cmd"}, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("100m")}, + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1")}, + }, + }, { + Name: "2nd-step", + Image: "image", + Command: []string{"cmd"}, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("200m")}, + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1")}, + }, + }}, + ComputeResources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1")}, + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("2")}, + }, + expectedComputeResources: []corev1.ResourceRequirements{{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("500m")}, + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("2")}, + }, { + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("500m")}, + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("2")}, + }}, + }, { + desc: "steps with partial compute resources are overridden by task-level compute resources", + Steps: []v1beta1.Step{{ + Name: "1st-step", + Image: "image", + Command: []string{"cmd"}, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("100m")}, + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1")}, + }, + }, { + Name: "2nd-step", + Image: "image", + Command: []string{"cmd"}, + }}, + ComputeResources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1")}, + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("2")}, + }, + expectedComputeResources: []corev1.ResourceRequirements{{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("500m")}, + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("2")}, + }, { + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("500m")}, + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("2")}, + }}, + }, { + desc: "steps with compute resources are preserved, if there are no task-level compute resources", + Steps: []v1beta1.Step{{ + Name: "1st-step", + Image: "image", + Command: []string{"cmd"}, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("100m")}, + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1")}, + }, + }, { + Name: "2nd-step", + Image: "image", + Command: []string{"cmd"}, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("200m")}, + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1")}, + }, + }}, + ComputeResources: corev1.ResourceRequirements{}, + expectedComputeResources: []corev1.ResourceRequirements{{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("100m")}, + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1")}, + }, { + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("200m")}, + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1")}, + }}, + }} + + for _, tc := range testcases { + t.Run(tc.desc, func(t *testing.T) { + tasklevel.ApplyTaskLevelComputeResources(tc.Steps, &tc.ComputeResources) + + if err := verifyTaskLevelComputeResources(tc.Steps, tc.expectedComputeResources); err != nil { + t.Errorf("verifyTaskLevelComputeResources: %v", err) + } + }) + } +} + +// verifyTaskLevelComputeResources verifies that the given TaskRun's containers have the expected compute resources. +func verifyTaskLevelComputeResources(steps []v1beta1.Step, expectedComputeResources []corev1.ResourceRequirements) error { + if len(expectedComputeResources) != len(steps) { + return fmt.Errorf("expected %d compute resource requirements, got %d", len(expectedComputeResources), len(steps)) + } + for id, step := range steps { + if d := cmp.Diff(expectedComputeResources[id], step.Resources); d != "" { + return fmt.Errorf("container \"#%d\" resource requirements don't match %s", id, diff.PrintWantGot(d)) + } + } + return nil +} diff --git a/pkg/pod/pod.go b/pkg/pod/pod.go index d25730af968..b4752863560 100644 --- a/pkg/pod/pod.go +++ b/pkg/pod/pod.go @@ -28,6 +28,7 @@ import ( "github.com/tektoncd/pipeline/pkg/apis/pipeline" "github.com/tektoncd/pipeline/pkg/apis/pipeline/pod" "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1beta1" + "github.com/tektoncd/pipeline/pkg/internal/computeresources/tasklevel" "github.com/tektoncd/pipeline/pkg/names" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -143,6 +144,9 @@ func (b *Builder) Build(ctx context.Context, taskRun *v1beta1.TaskRun, taskSpec if err != nil { return nil, err } + if alphaAPIEnabled && taskRun.Spec.ComputeResources != nil { + tasklevel.ApplyTaskLevelComputeResources(steps, taskRun.Spec.ComputeResources) + } sidecars, err := v1beta1.MergeSidecarsWithOverrides(taskSpec.Sidecars, taskRun.Spec.SidecarOverrides) if err != nil { return nil, err diff --git a/pkg/pod/pod_test.go b/pkg/pod/pod_test.go index c94d990c818..ee9bcc31e71 100644 --- a/pkg/pod/pod_test.go +++ b/pkg/pod/pod_test.go @@ -2078,6 +2078,217 @@ debug-fail-continue-heredoc-randomly-generated-mz4c7 } } +type ExpectedComputeResources struct { + name string + ResourceRequirements corev1.ResourceRequirements +} + +func TestPodBuild_TaskLevelResourceRequirements(t *testing.T) { + testcases := []struct { + desc string + ts v1beta1.TaskSpec + trs v1beta1.TaskRunSpec + expectedComputeResources []ExpectedComputeResources + }{{ + desc: "overwrite stepTemplate resources requirements", + ts: v1beta1.TaskSpec{ + Steps: []v1beta1.Step{{ + Name: "1st-step", + Image: "image", + Command: []string{"cmd"}, + }, { + Name: "2nd-step", + Image: "image", + Command: []string{"cmd"}, + }}, + StepTemplate: &v1beta1.StepTemplate{ + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceMemory: resource.MustParse("500Mi"), + }, + }, + }, + }, + trs: v1beta1.TaskRunSpec{ + ComputeResources: &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("2"), + corev1.ResourceMemory: resource.MustParse("2Gi"), + }, + }, + }, + expectedComputeResources: []ExpectedComputeResources{{ + name: "step-1st-step", + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceMemory: resource.MustParse("1Gi"), + }, + }, + }, { + name: "step-2nd-step", + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceMemory: resource.MustParse("1Gi"), + }, + }, + }}, + }, { + desc: "overwrite step resources requirements", + ts: v1beta1.TaskSpec{ + Steps: []v1beta1.Step{{ + Name: "1st-step", + Image: "image", + Command: []string{"cmd"}, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("250m"), + corev1.ResourceMemory: resource.MustParse("500Mi"), + }, + }, + }, { + Name: "2nd-step", + Image: "image", + Command: []string{"cmd"}, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("300m"), + corev1.ResourceMemory: resource.MustParse("500Mi"), + }, + }, + }}, + }, + trs: v1beta1.TaskRunSpec{ + ComputeResources: &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("2"), + corev1.ResourceMemory: resource.MustParse("2Gi"), + }, + }, + }, + expectedComputeResources: []ExpectedComputeResources{{ + name: "step-1st-step", + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceMemory: resource.MustParse("1Gi"), + }, + }, + }, { + name: "step-2nd-step", + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceMemory: resource.MustParse("1Gi"), + }, + }, + }}, + }, { + desc: "with sidecar resource requirements", + ts: v1beta1.TaskSpec{ + Steps: []v1beta1.Step{{ + Name: "1st-step", + Image: "image", + Command: []string{"cmd"}, + }}, + Sidecars: []v1beta1.Sidecar{{ + Name: "sidecar", + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("750m"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1.5"), + }, + }, + }}, + }, + trs: v1beta1.TaskRunSpec{ + ComputeResources: &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("2"), + corev1.ResourceMemory: resource.MustParse("2Gi"), + }, + }, + }, + expectedComputeResources: []ExpectedComputeResources{{ + name: "step-1st-step", + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("2"), + corev1.ResourceMemory: resource.MustParse("2Gi"), + }, + }, + }, { + name: "sidecar-sidecar", + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("750m"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1.5"), + }, + }, + }}, + }} + + for _, tc := range testcases { + t.Run(tc.desc, func(t *testing.T) { + names.TestingSeed() + store := config.NewStore(logtesting.TestLogger(t)) + enableAlphaAPI := map[string]string{"enable-api-fields": "alpha"} + store.OnConfigChanged( + &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: config.GetFeatureFlagsConfigName(), Namespace: system.Namespace()}, + Data: enableAlphaAPI, + }, + ) + + kubeclient := fakek8s.NewSimpleClientset( + &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "default", Namespace: "default"}}, + ) + builder := Builder{ + Images: images, + KubeClient: kubeclient, + } + tr := &v1beta1.TaskRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo-taskrun", + Namespace: "default", + }, + Spec: tc.trs, + } + + gotPod, err := builder.Build(store.ToContext(context.Background()), tr, tc.ts) + if err != nil { + t.Fatalf("builder.Build: %v", err) + } + + if err := verifyTaskLevelComputeResources(tc.expectedComputeResources, gotPod.Spec.Containers); err != nil { + t.Errorf("verifyTaskLevelComputeResources: %v", err) + } + }) + } +} + +// verifyTaskLevelComputeResources verifies that the given TaskRun's containers have the expected compute resources. +func verifyTaskLevelComputeResources(expectedComputeResources []ExpectedComputeResources, containers []corev1.Container) error { + if len(expectedComputeResources) != len(containers) { + return fmt.Errorf("expected %d compute resource requirements, got %d", len(expectedComputeResources), len(containers)) + } + for i, r := range expectedComputeResources { + if r.name != containers[i].Name { + return fmt.Errorf("expected container name %s, got %s", r.name, containers[i].Name) + } + if d := cmp.Diff(r.ResourceRequirements, containers[i].Resources); d != "" { + return fmt.Errorf("container \"#%d\" resource requirements don't match %s", i, diff.PrintWantGot(d)) + } + } + return nil +} + func TestMakeLabels(t *testing.T) { taskRunName := "task-run-name" want := map[string]string{