diff --git a/docs/resources.md b/docs/resources.md index 80b433740b3..1bcfd536183 100644 --- a/docs/resources.md +++ b/docs/resources.md @@ -377,6 +377,22 @@ spec: value: refs/pull/52525/head ``` +#### Setting Env. Variables + +`Env` section can be used to set common proxy environment variables or any arbitrary variables in a container where +`git` commands are executed: + +```yaml +spec: + type: git + params: + - name: url + value: https://github.com/bobcatfish/wizzbang.git + env: + - name: HTTPS_PROXY + value: "myproxy.com" +``` + ### Pull Request Resource The `pullRequest` resource represents a pull request event from a source control diff --git a/examples/v1alpha1/taskruns/git-resource.yaml b/examples/v1alpha1/taskruns/git-resource.yaml index 04b91e280f6..4359a41e1f7 100644 --- a/examples/v1alpha1/taskruns/git-resource.yaml +++ b/examples/v1alpha1/taskruns/git-resource.yaml @@ -69,3 +69,59 @@ spec: value: pull/2932/head - name: url value: https://github.com/GoogleContainerTools/skaffold + +--- +apiVersion: tekton.dev/v1alpha1 +kind: TaskRun +metadata: + generateName: git-resource-sslverify- +spec: + taskSpec: + inputs: + resources: + - name: skaffold + type: git + steps: + - image: ubuntu + script: cat skaffold/README.md + inputs: + resources: + - name: skaffold + resourceSpec: + type: git + params: + - name: revision + value: master + - name: url + value: https://github.com/GoogleContainerTools/skaffold + - name: sslverify + value: "false" +--- + +apiVersion: tekton.dev/v1alpha1 +kind: TaskRun +metadata: + generateName: git-resource-no-proxy- +spec: + taskSpec: + inputs: + resources: + - name: skaffold + type: git + steps: + - image: ubuntu + script: cat skaffold/README.md + inputs: + resources: + - name: skaffold + resourceSpec: + type: git + params: + - name: revision + value: master + - name: url + value: https://github.com/GoogleContainerTools/skaffold + env: + - name: NO_PROXY + value: "google.com" +--- diff --git a/pkg/apis/resource/v1alpha1/git/git_resource.go b/pkg/apis/resource/v1alpha1/git/git_resource.go index 82c11c8f302..94b24c6b229 100644 --- a/pkg/apis/resource/v1alpha1/git/git_resource.go +++ b/pkg/apis/resource/v1alpha1/git/git_resource.go @@ -28,6 +28,10 @@ import ( corev1 "k8s.io/api/core/v1" ) +const ( + TektonResourceName = "TEKTON_RESOURCE_NAME" +) + var ( gitSource = "git-source" ) @@ -44,9 +48,10 @@ type Resource struct { Revision string `json:"revision"` Submodules bool `json:"submodules"` - Depth uint `json:"depth"` - SSLVerify bool `json:"sslVerify"` - GitImage string `json:"-"` + Depth uint `json:"depth"` + SSLVerify bool `json:"sslVerify"` + GitImage string `json:"-"` + Env []corev1.EnvVar `json:"-"` } // NewResource creates a new git resource to pass to a Task @@ -61,6 +66,7 @@ func NewResource(gitImage string, r *resource.PipelineResource) (*Resource, erro Submodules: true, Depth: 1, SSLVerify: true, + Env: r.Spec.Env, } for _, param := range r.Spec.Params { switch { @@ -147,6 +153,13 @@ func (s *Resource) GetInputTaskModifier(_ *v1alpha1.TaskSpec, path string) (v1al args = append(args, "-sslVerify=false") } + if !s.isTektonResourceName() { + s.Env = append(s.Env, corev1.EnvVar{ + Name: TektonResourceName, + Value: s.Name, + }) + } + step := v1alpha1.Step{ Container: corev1.Container{ Name: names.SimpleNameGenerator.RestrictLengthWithRandomSuffix(gitSource + "-" + s.Name), @@ -155,10 +168,7 @@ func (s *Resource) GetInputTaskModifier(_ *v1alpha1.TaskSpec, path string) (v1al Args: args, WorkingDir: pipeline.WorkspaceDir, // This is used to populate the ResourceResult status. - Env: []corev1.EnvVar{{ - Name: "TEKTON_RESOURCE_NAME", - Value: s.Name, - }}, + Env: s.Env, }, } @@ -171,3 +181,12 @@ func (s *Resource) GetInputTaskModifier(_ *v1alpha1.TaskSpec, path string) (v1al func (s *Resource) GetOutputTaskModifier(_ *v1alpha1.TaskSpec, _ string) (v1alpha1.TaskModifier, error) { return &v1alpha1.InternalTaskModifier{}, nil } + +func (s *Resource) isTektonResourceName() bool { + for _, e := range s.Env { + if e.Name == TektonResourceName { + return true + } + } + return false +} diff --git a/pkg/apis/resource/v1alpha1/git/git_resource_test.go b/pkg/apis/resource/v1alpha1/git/git_resource_test.go index 5a7cad8cd19..7dbb4af9723 100644 --- a/pkg/apis/resource/v1alpha1/git/git_resource_test.go +++ b/pkg/apis/resource/v1alpha1/git/git_resource_test.go @@ -329,6 +329,45 @@ func TestGitResource_GetDownloadTaskModifier(t *testing.T) { WorkingDir: "/workspace", Env: []corev1.EnvVar{{Name: "TEKTON_RESOURCE_NAME", Value: "git-resource"}}, }, + }, { + desc: "With Env", + gitResource: &git.Resource{ + Name: "git-resource", + Type: resourcev1alpha1.PipelineResourceTypeGit, + URL: "git@github.com:test/test.git", + Revision: "master", + GitImage: "override-with-git:latest", + Submodules: false, + Depth: 1, + SSLVerify: false, + Env: []corev1.EnvVar{{ + Name: "FOO", + Value: "foo", + ValueFrom: nil, + }, { + Name: "BAR", + Value: "bar", + ValueFrom: nil, + }}, + }, + want: corev1.Container{ + Name: "git-source-git-resource-6nl7g", + Image: "override-with-git:latest", + Command: []string{"/ko-app/git-init"}, + Args: []string{ + "-url", + "git@github.com:test/test.git", + "-revision", + "master", + "-path", + "/test/test", + "-submodules=false", + "-sslVerify=false", + }, + WorkingDir: "/workspace", + Env: []corev1.EnvVar{{Name: "FOO", Value: "foo"}, {Name: "BAR", Value: "bar"}, + {Name: "TEKTON_RESOURCE_NAME", Value: "git-resource"}}, + }, }} { t.Run(tc.desc, func(t *testing.T) { ts := pipelinev1alpha1.TaskSpec{} diff --git a/pkg/apis/resource/v1alpha1/pipeline_resource_types.go b/pkg/apis/resource/v1alpha1/pipeline_resource_types.go index 3d6cc12fc75..8c7b8d89caa 100644 --- a/pkg/apis/resource/v1alpha1/pipeline_resource_types.go +++ b/pkg/apis/resource/v1alpha1/pipeline_resource_types.go @@ -17,6 +17,7 @@ limitations under the License. package v1alpha1 import ( + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -102,6 +103,7 @@ type PipelineResourceSpec struct { // Secrets to fetch to populate some of resource fields // +optional SecretParams []SecretParam `json:"secrets,omitempty"` + Env []v1.EnvVar `json:"env,omitempty"` } // SecretParam indicates which secret can be used to populate a field of the resource diff --git a/pkg/reconciler/pipelinerun/resources/input_output_steps.go b/pkg/reconciler/pipelinerun/resources/input_output_steps.go index 0079f7b1b5d..b95e37fc967 100644 --- a/pkg/reconciler/pipelinerun/resources/input_output_steps.go +++ b/pkg/reconciler/pipelinerun/resources/input_output_steps.go @@ -79,6 +79,11 @@ func GetInputSteps(inputs map[string]*v1alpha1.PipelineResource, inputResources } } + // set Env. when resource is of type Git + if inputResource.Spec.Type == v1alpha1.PipelineResourceTypeGit { + taskInputResource.ResourceSpec.Env = inputResource.Spec.Env + } + // Determine if the value is meant to come `from` a previous Task - if so, add the path to the pvc // that contains the data as the `path` the resulting TaskRun should get the data from. var stepSourceNames []string diff --git a/pkg/reconciler/pipelinerun/resources/input_output_steps_test.go b/pkg/reconciler/pipelinerun/resources/input_output_steps_test.go index 13d6eac458f..61bdde5a687 100644 --- a/pkg/reconciler/pipelinerun/resources/input_output_steps_test.go +++ b/pkg/reconciler/pipelinerun/resources/input_output_steps_test.go @@ -19,6 +19,8 @@ import ( "sort" "testing" + v1 "k8s.io/api/core/v1" + "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1alpha1" @@ -147,6 +149,17 @@ func TestGetInputSteps(t *testing.T) { SecretParams: nil, }, } + r3 := &v1alpha1.PipelineResource{ + Spec: v1alpha1.PipelineResourceSpec{ + Type: v1alpha1.PipelineResourceTypeGit, + Params: []v1alpha1.ResourceParam{{ + Name: "url", + Value: "https://github.com/tektoncd/pipeline.git", + }}, + SecretParams: nil, + Env: []v1.EnvVar{{Name: "FOO", Value: "foo"}}, + }, + } tcs := []struct { name string inputs map[string]*v1alpha1.PipelineResource @@ -259,6 +272,23 @@ func TestGetInputSteps(t *testing.T) { }, Paths: []string{"/pvc/prev-task-1/test-input", "/pvc/prev-task-2/test-input"}, }}, + }, { + name: "task-with-git-input-resource-with-env", + inputs: map[string]*v1alpha1.PipelineResource{"test-input": r3}, + expectedtaskInputResources: []v1alpha1.TaskResourceBinding{{ + PipelineResourceBinding: v1alpha1.PipelineResourceBinding{ + ResourceSpec: &r3.Spec, + Name: "test-input", + }, + }}, + pipelineTask: &v1alpha1.PipelineTask{ + Name: "sample-test-task", + Resources: &v1alpha1.PipelineTaskResources{ + Inputs: []v1alpha1.PipelineTaskInputResource{{ + Name: "test-input", + }}, + }, + }, }, } for _, tc := range tcs { diff --git a/test/builder/pipeline.go b/test/builder/pipeline.go index a8f7615634d..0c04f3159c1 100644 --- a/test/builder/pipeline.go +++ b/test/builder/pipeline.go @@ -541,6 +541,13 @@ func PipelineResourceSpecParam(name, value string) PipelineResourceSpecOp { } } +// PipelineResourceSpecEnv adds a ResourceEnv, with specified name and value, to the PipelineResourceSpec. +func PipelineResourceSpecEnv(env []corev1.EnvVar) PipelineResourceSpecOp { + return func(spec *v1alpha1.PipelineResourceSpec) { + spec.Env = append(spec.Env, env...) + } +} + // PipelineResourceSpecSecretParam adds a SecretParam, with specified fieldname, secretKey and secretName, to the PipelineResourceSpec. func PipelineResourceSpecSecretParam(fieldname, secretName, secretKey string) PipelineResourceSpecOp { return func(spec *v1alpha1.PipelineResourceSpec) { diff --git a/test/git_checkout_test.go b/test/git_checkout_test.go index bf81c2deb85..9d97b28e065 100644 --- a/test/git_checkout_test.go +++ b/test/git_checkout_test.go @@ -51,7 +51,7 @@ func TestGitPipelineRun(t *testing.T) { defer tearDown(t, c, namespace) t.Logf("Creating Git PipelineResource %s", gitSourceResourceName) - if _, err := c.PipelineResourceClient.Create(getGitPipelineResource(namespace, revision)); err != nil { + if _, err := c.PipelineResourceClient.Create(getGitPipelineResource(namespace, revision, "true", []corev1.EnvVar{})); err != nil { t.Fatalf("Failed to create Pipeline Resource `%s`: %s", gitSourceResourceName, err) } @@ -78,6 +78,40 @@ func TestGitPipelineRun(t *testing.T) { } } +// TestGitPipelineRun_Disable_SSLVerify will verify the source code is retrieved even after disabling SSL certificates (sslVerify) +func TestGitPipelineRun_Disable_SSLVerify(t *testing.T) { + t.Parallel() + + c, namespace := setup(t) + knativetest.CleanupOnInterrupt(func() { tearDown(t, c, namespace) }, t.Logf) + defer tearDown(t, c, namespace) + + t.Logf("Creating Git PipelineResource %s", gitSourceResourceName) + if _, err := c.PipelineResourceClient.Create(getGitPipelineResource(namespace, "master", "false", []corev1.EnvVar{})); err != nil { + t.Fatalf("Failed to create Pipeline Resource `%s`: %s", gitSourceResourceName, err) + } + + t.Logf("Creating Task %s", gitTestTaskName) + if _, err := c.TaskClient.Create(getGitCheckTask(namespace)); err != nil { + t.Fatalf("Failed to create Task `%s`: %s", gitTestTaskName, err) + } + + t.Logf("Creating Pipeline %s", gitTestPipelineName) + if _, err := c.PipelineClient.Create(getGitCheckPipeline(namespace)); err != nil { + t.Fatalf("Failed to create Pipeline `%s`: %s", gitTestPipelineName, err) + } + + t.Logf("Creating PipelineRun %s", gitTestPipelineRunName) + if _, err := c.PipelineRunClient.Create(getGitCheckPipelineRun(namespace)); err != nil { + t.Fatalf("Failed to create Pipeline `%s`: %s", gitTestPipelineRunName, err) + } + + if err := WaitForPipelineRunState(c, gitTestPipelineRunName, timeout, PipelineRunSucceed(gitTestPipelineRunName), "PipelineRunCompleted"); err != nil { + t.Errorf("Error waiting for PipelineRun %s to finish: %s", gitTestPipelineRunName, err) + t.Fatalf("PipelineRun execution failed") + } +} + // TestGitPipelineRunFail is a test to ensure that the code extraction from github fails as expected when // an invalid revision is passed on the pipelineresource. func TestGitPipelineRunFail(t *testing.T) { @@ -88,7 +122,7 @@ func TestGitPipelineRunFail(t *testing.T) { defer tearDown(t, c, namespace) t.Logf("Creating Git PipelineResource %s", gitSourceResourceName) - if _, err := c.PipelineResourceClient.Create(getGitPipelineResource(namespace, "Idontexistrabbitmonkeydonkey")); err != nil { + if _, err := c.PipelineResourceClient.Create(getGitPipelineResource(namespace, "Idontexistrabbitmonkeydonkey", "true", []corev1.EnvVar{})); err != nil { t.Fatalf("Failed to create Pipeline Resource `%s`: %s", gitSourceResourceName, err) } @@ -146,11 +180,86 @@ func TestGitPipelineRunFail(t *testing.T) { } } -func getGitPipelineResource(namespace, revision string) *v1alpha1.PipelineResource { +// TestGitPipelineRunFail_HTTPS_PROXY is a test to ensure that the code extraction from github fails as expected when +// an invalid HTTPS_PROXY is set in Env section of the pipelineresource. +func TestGitPipelineRunFail_HTTPS_PROXY(t *testing.T) { + t.Parallel() + + c, namespace := setup(t) + knativetest.CleanupOnInterrupt(func() { tearDown(t, c, namespace) }, t.Logf) + defer tearDown(t, c, namespace) + + env := []corev1.EnvVar{{ + Name: "HTTPS_PROXY", + Value: "invalid.https.proxy.com", + }} + + t.Logf("Creating Git PipelineResource %s", gitSourceResourceName) + if _, err := c.PipelineResourceClient.Create(getGitPipelineResource(namespace, "master", "true", env)); err != nil { + t.Fatalf("Failed to create Pipeline Resource `%s`: %s", gitSourceResourceName, err) + } + + t.Logf("Creating Task %s", gitTestTaskName) + if _, err := c.TaskClient.Create(getGitCheckTask(namespace)); err != nil { + t.Fatalf("Failed to create Task `%s`: %s", gitTestTaskName, err) + } + + t.Logf("Creating Pipeline %s", gitTestPipelineName) + if _, err := c.PipelineClient.Create(getGitCheckPipeline(namespace)); err != nil { + t.Fatalf("Failed to create Pipeline `%s`: %s", gitTestPipelineName, err) + } + + t.Logf("Creating PipelineRun %s", gitTestPipelineRunName) + if _, err := c.PipelineRunClient.Create(getGitCheckPipelineRun(namespace)); err != nil { + t.Fatalf("Failed to create Pipeline `%s`: %s", gitTestPipelineRunName, err) + } + + if err := WaitForPipelineRunState(c, gitTestPipelineRunName, timeout, PipelineRunSucceed(gitTestPipelineRunName), "PipelineRunCompleted"); err != nil { + taskruns, err := c.TaskRunClient.List(metav1.ListOptions{}) + if err != nil { + t.Errorf("Error getting TaskRun list for PipelineRun %s %s", gitTestPipelineRunName, err) + } + for _, tr := range taskruns.Items { + if tr.Status.PodName != "" { + p, err := c.KubeClient.Kube.CoreV1().Pods(namespace).Get(tr.Status.PodName, metav1.GetOptions{}) + if err != nil { + t.Fatalf("Error getting pod `%s` in namespace `%s`", tr.Status.PodName, namespace) + } + + for _, stat := range p.Status.ContainerStatuses { + if strings.HasPrefix(stat.Name, "step-git-source-"+gitSourceResourceName) { + if stat.State.Terminated != nil { + req := c.KubeClient.Kube.CoreV1().Pods(namespace).GetLogs(p.Name, &corev1.PodLogOptions{Container: stat.Name}) + logContent, err := req.Do().Raw() + if err != nil { + t.Fatalf("Error getting pod logs for pod `%s` and container `%s` in namespace `%s`", tr.Status.PodName, stat.Name, namespace) + } + // Check for failure messages from fetch and pull in the log file + if strings.Contains(strings.ToLower(string(logContent)), "could not resolve proxy: invalid.https.proxy.com") && + strings.Contains(strings.ToLower(string(logContent)), "pathspec 'master' did not match any file(s) known to git") { + t.Logf("Found exepected errors when using non-existent https proxy") + } else { + t.Logf("Container `%s` log File: %s", stat.Name, logContent) + t.Fatalf("The git code extraction did not fail as expected. Expected errors not found in log file.") + } + } + } + } + } + } + + } else { + t.Fatalf("PipelineRun succeeded when should have failed") + } +} + +func getGitPipelineResource(namespace, revision, sslverify string, env []corev1.EnvVar) *v1alpha1.PipelineResource { return tb.PipelineResource(gitSourceResourceName, namespace, tb.PipelineResourceSpec( v1alpha1.PipelineResourceTypeGit, tb.PipelineResourceSpecParam("Url", "https://github.com/tektoncd/pipeline"), tb.PipelineResourceSpecParam("Revision", revision), + tb.PipelineResourceSpecParam("sslVerify", sslverify), + tb.PipelineResourceSpecEnv(env), )) }