From bf83a8ad467e518fe9597e692091421b18a3a76b Mon Sep 17 00:00:00 2001 From: Priti Desai Date: Sat, 7 Mar 2020 14:37:10 -0800 Subject: [PATCH] introducing env to git pipeline resource Along with type and params, we can specify env. now for git resources. Env. takes a list of envionment variable names and their values and sets those variables in a container where git CLI is being executed e.g.: ``` inputs: resources: - name: skaffold resourceSpec: type: git params: - name: revision value: master - name: url value: https://github.com/GoogleContainerTools/skaffold env: - name: HTTPS_PROXY value: "myproxy.com" ``` --- docs/resources.md | 32 +++++ examples/v1alpha1/taskruns/git-resource.yaml | 55 +++++++++ .../resource/v1alpha1/git/git_resource.go | 30 +++-- .../v1alpha1/git/git_resource_test.go | 30 +++-- pkg/git/git.go | 34 +++++- test/git_checkout_test.go | 112 +++++++++++++++++- 6 files changed, 264 insertions(+), 29 deletions(-) diff --git a/docs/resources.md b/docs/resources.md index 80b433740b3..2b3ffb061c8 100644 --- a/docs/resources.md +++ b/docs/resources.md @@ -377,6 +377,38 @@ spec: value: refs/pull/52525/head ``` +#### Using HTTP/HTTPS Proxy + +The `httpProxy` and `httpsProxy` parameter can be used to proxy non-SSL/SSL requests, for example to use an enterprise +proxy server for SSL requests: + +```yaml +spec: + type: git + params: + - name: url + value: https://github.com/bobcatfish/wizzbang.git + - name: httpsProxy + value: "my-enterprise.proxy.com" +``` + +#### Using No Proxy + +The `noProxy` parameter can be used to opt out of proxying, for example, to not proxy HTTP/HTTPS requests to +`no.proxy.com`: + +```yaml +spec: + type: git + params: + - name: url + value: https://github.com/bobcatfish/wizzbang.git + - name: noProxy + value: "no.proxy.com" +``` + +Note: `httpProxy`, `httpsProxy`, and `noProxy` are all optional but no validation done if all three are specified. + ### Pull Request Resource The `pullRequest` resource represents a pull request event from a source control diff --git a/examples/v1alpha1/taskruns/git-resource.yaml b/examples/v1alpha1/taskruns/git-resource.yaml index 04b91e280f6..d3440a75b6d 100644 --- a/examples/v1alpha1/taskruns/git-resource.yaml +++ b/examples/v1alpha1/taskruns/git-resource.yaml @@ -69,3 +69,58 @@ spec: value: pull/2932/head - name: url value: https://github.com/GoogleContainerTools/skaffold + +--- +apiVersion: tekton.dev/v1alpha1 +kind: TaskRun +metadata: + generateName: git-resource-sslverify- +spec: + taskSpec: + inputs: + resources: + - name: skaffold + type: git + steps: + - image: ubuntu + script: cat skaffold/README.md + inputs: + resources: + - name: skaffold + resourceSpec: + type: git + params: + - name: revision + value: master + - name: url + value: https://github.com/GoogleContainerTools/skaffold + - name: sslVerify + value: "false" +--- + +apiVersion: tekton.dev/v1alpha1 +kind: TaskRun +metadata: + generateName: git-resource-no-proxy- +spec: + taskSpec: + inputs: + resources: + - name: skaffold + type: git + steps: + - image: ubuntu + script: cat skaffold/README.md + inputs: + resources: + - name: skaffold + resourceSpec: + type: git + params: + - name: revision + value: master + - name: url + value: https://github.com/GoogleContainerTools/skaffold + - name: noProxy + value: "google.com" +--- diff --git a/pkg/apis/resource/v1alpha1/git/git_resource.go b/pkg/apis/resource/v1alpha1/git/git_resource.go index 82c11c8f302..bced563c471 100644 --- a/pkg/apis/resource/v1alpha1/git/git_resource.go +++ b/pkg/apis/resource/v1alpha1/git/git_resource.go @@ -44,9 +44,12 @@ type Resource struct { Revision string `json:"revision"` Submodules bool `json:"submodules"` - Depth uint `json:"depth"` - SSLVerify bool `json:"sslVerify"` - GitImage string `json:"-"` + Depth uint `json:"depth"` + SSLVerify bool `json:"sslVerify"` + HTTPProxy string `json:"httpProxy"` + HTTPSProxy string `json:"httpsProxy"` + NOProxy string `json:"noProxy"` + GitImage string `json:"-"` } // NewResource creates a new git resource to pass to a Task @@ -74,6 +77,12 @@ func NewResource(gitImage string, r *resource.PipelineResource) (*Resource, erro gitResource.Depth = toUint(param.Value, 1) case strings.EqualFold(param.Name, "SSLVerify"): gitResource.SSLVerify = toBool(param.Value, true) + case strings.EqualFold(param.Name, "HTTPProxy"): + gitResource.HTTPProxy = param.Value + case strings.EqualFold(param.Name, "HTTPSProxy"): + gitResource.HTTPSProxy = param.Value + case strings.EqualFold(param.Name, "NOProxy"): + gitResource.NOProxy = param.Value } } // default revision to master if nothing is provided @@ -120,12 +129,15 @@ func (s *Resource) GetURL() string { // Replacements is used for template replacement on a GitResource inside of a Taskrun. func (s *Resource) Replacements() map[string]string { return map[string]string{ - "name": s.Name, - "type": s.Type, - "url": s.URL, - "revision": s.Revision, - "depth": strconv.FormatUint(uint64(s.Depth), 10), - "sslVerify": strconv.FormatBool(s.SSLVerify), + "name": s.Name, + "type": s.Type, + "url": s.URL, + "revision": s.Revision, + "depth": strconv.FormatUint(uint64(s.Depth), 10), + "sslVerify": strconv.FormatBool(s.SSLVerify), + "httpProxy": s.HTTPProxy, + "httpsProxy": s.HTTPSProxy, + "noProxy": s.NOProxy, } } diff --git a/pkg/apis/resource/v1alpha1/git/git_resource_test.go b/pkg/apis/resource/v1alpha1/git/git_resource_test.go index 5a7cad8cd19..74a498dfdc0 100644 --- a/pkg/apis/resource/v1alpha1/git/git_resource_test.go +++ b/pkg/apis/resource/v1alpha1/git/git_resource_test.go @@ -185,21 +185,27 @@ func TestNewGitResource_Valid(t *testing.T) { func TestGitResource_Replacements(t *testing.T) { r := &git.Resource{ - Name: "git-resource", - Type: resourcev1alpha1.PipelineResourceTypeGit, - URL: "git@github.com:test/test.git", - Revision: "master", - Depth: 16, - SSLVerify: false, + Name: "git-resource", + Type: resourcev1alpha1.PipelineResourceTypeGit, + URL: "git@github.com:test/test.git", + Revision: "master", + Depth: 16, + SSLVerify: false, + HTTPProxy: "http-proxy.git.com", + HTTPSProxy: "https-proxy.git.com", + NOProxy: "*", } want := map[string]string{ - "name": "git-resource", - "type": string(resourcev1alpha1.PipelineResourceTypeGit), - "url": "git@github.com:test/test.git", - "revision": "master", - "depth": "16", - "sslVerify": "false", + "name": "git-resource", + "type": string(resourcev1alpha1.PipelineResourceTypeGit), + "url": "git@github.com:test/test.git", + "revision": "master", + "depth": "16", + "sslVerify": "false", + "httpProxy": "http-proxy.git.com", + "httpsProxy": "https-proxy.git.com", + "noProxy": "*", } got := r.Replacements() diff --git a/pkg/git/git.go b/pkg/git/git.go index 77364fe07e8..43bea642386 100644 --- a/pkg/git/git.go +++ b/pkg/git/git.go @@ -46,11 +46,14 @@ func run(logger *zap.SugaredLogger, dir string, args ...string) (string, error) // FetchSpec describes how to initialize and fetch from a Git repository. type FetchSpec struct { - URL string - Revision string - Path string - Depth uint - SSLVerify bool + URL string + Revision string + Path string + Depth uint + SSLVerify bool + HTTPProxy string + HTTPSProxy string + NOProxy string } // Fetch fetches the specified git repository at the revision into path. @@ -81,6 +84,27 @@ func Fetch(logger *zap.SugaredLogger, spec FetchSpec) error { return err } + if len(spec.HTTPProxy) != 0 { + if _, err := run(logger, "", "config", "--global", "http.proxy", spec.HTTPProxy); err != nil { + logger.Warnf("Failed to set http.proxy in git config: %s", err) + return err + } + } + + if len(spec.HTTPSProxy) != 0 { + if _, err := run(logger, "", "config", "--global", "https.proxy", spec.HTTPSProxy); err != nil { + logger.Warnf("Failed to set https.proxy in git config: %s", err) + return err + } + } + + if len(spec.NOProxy) != 0 { + if _, err := run(logger, "", "config", "--global", "no.proxy", spec.NOProxy); err != nil { + logger.Warnf("Failed to set no.proxy in git config: %s", err) + return err + } + } + fetchArgs := []string{"fetch", "--recurse-submodules=yes"} if spec.Depth > 0 { fetchArgs = append(fetchArgs, fmt.Sprintf("--depth=%d", spec.Depth)) diff --git a/test/git_checkout_test.go b/test/git_checkout_test.go index bf81c2deb85..c90444f716b 100644 --- a/test/git_checkout_test.go +++ b/test/git_checkout_test.go @@ -51,7 +51,7 @@ func TestGitPipelineRun(t *testing.T) { defer tearDown(t, c, namespace) t.Logf("Creating Git PipelineResource %s", gitSourceResourceName) - if _, err := c.PipelineResourceClient.Create(getGitPipelineResource(namespace, revision)); err != nil { + if _, err := c.PipelineResourceClient.Create(getGitPipelineResource(namespace, revision, "true", "", "", "")); err != nil { t.Fatalf("Failed to create Pipeline Resource `%s`: %s", gitSourceResourceName, err) } @@ -78,6 +78,40 @@ func TestGitPipelineRun(t *testing.T) { } } +// TestGitPipelineRun_Disable_SSLVerify will verify the source code is retrieved even after disabling SSL certificates (sslVerify) +func TestGitPipelineRun_Disable_SSLVerify(t *testing.T) { + t.Parallel() + + c, namespace := setup(t) + knativetest.CleanupOnInterrupt(func() { tearDown(t, c, namespace) }, t.Logf) + defer tearDown(t, c, namespace) + + t.Logf("Creating Git PipelineResource %s", gitSourceResourceName) + if _, err := c.PipelineResourceClient.Create(getGitPipelineResource(namespace, "master", "false", "", "", "")); err != nil { + t.Fatalf("Failed to create Pipeline Resource `%s`: %s", gitSourceResourceName, err) + } + + t.Logf("Creating Task %s", gitTestTaskName) + if _, err := c.TaskClient.Create(getGitCheckTask(namespace)); err != nil { + t.Fatalf("Failed to create Task `%s`: %s", gitTestTaskName, err) + } + + t.Logf("Creating Pipeline %s", gitTestPipelineName) + if _, err := c.PipelineClient.Create(getGitCheckPipeline(namespace)); err != nil { + t.Fatalf("Failed to create Pipeline `%s`: %s", gitTestPipelineName, err) + } + + t.Logf("Creating PipelineRun %s", gitTestPipelineRunName) + if _, err := c.PipelineRunClient.Create(getGitCheckPipelineRun(namespace)); err != nil { + t.Fatalf("Failed to create Pipeline `%s`: %s", gitTestPipelineRunName, err) + } + + if err := WaitForPipelineRunState(c, gitTestPipelineRunName, timeout, PipelineRunSucceed(gitTestPipelineRunName), "PipelineRunCompleted"); err != nil { + t.Errorf("Error waiting for PipelineRun %s to finish: %s", gitTestPipelineRunName, err) + t.Fatalf("PipelineRun execution failed") + } +} + // TestGitPipelineRunFail is a test to ensure that the code extraction from github fails as expected when // an invalid revision is passed on the pipelineresource. func TestGitPipelineRunFail(t *testing.T) { @@ -88,7 +122,7 @@ func TestGitPipelineRunFail(t *testing.T) { defer tearDown(t, c, namespace) t.Logf("Creating Git PipelineResource %s", gitSourceResourceName) - if _, err := c.PipelineResourceClient.Create(getGitPipelineResource(namespace, "Idontexistrabbitmonkeydonkey")); err != nil { + if _, err := c.PipelineResourceClient.Create(getGitPipelineResource(namespace, "Idontexistrabbitmonkeydonkey", "true", "", "", "")); err != nil { t.Fatalf("Failed to create Pipeline Resource `%s`: %s", gitSourceResourceName, err) } @@ -146,11 +180,83 @@ func TestGitPipelineRunFail(t *testing.T) { } } -func getGitPipelineResource(namespace, revision string) *v1alpha1.PipelineResource { +// TestGitPipelineRunFail_HTTPS_PROXY is a test to ensure that the code extraction from github fails as expected when +// an invalid HTTPS_PROXY is passed on the pipelineresource. +func TestGitPipelineRunFail_HTTPS_PROXY(t *testing.T) { + t.Parallel() + + c, namespace := setup(t) + knativetest.CleanupOnInterrupt(func() { tearDown(t, c, namespace) }, t.Logf) + defer tearDown(t, c, namespace) + + t.Logf("Creating Git PipelineResource %s", gitSourceResourceName) + if _, err := c.PipelineResourceClient.Create(getGitPipelineResource(namespace, "master", "true", "", "invalid.https.proxy.com", "")); err != nil { + t.Fatalf("Failed to create Pipeline Resource `%s`: %s", gitSourceResourceName, err) + } + + t.Logf("Creating Task %s", gitTestTaskName) + if _, err := c.TaskClient.Create(getGitCheckTask(namespace)); err != nil { + t.Fatalf("Failed to create Task `%s`: %s", gitTestTaskName, err) + } + + t.Logf("Creating Pipeline %s", gitTestPipelineName) + if _, err := c.PipelineClient.Create(getGitCheckPipeline(namespace)); err != nil { + t.Fatalf("Failed to create Pipeline `%s`: %s", gitTestPipelineName, err) + } + + t.Logf("Creating PipelineRun %s", gitTestPipelineRunName) + if _, err := c.PipelineRunClient.Create(getGitCheckPipelineRun(namespace)); err != nil { + t.Fatalf("Failed to create Pipeline `%s`: %s", gitTestPipelineRunName, err) + } + + if err := WaitForPipelineRunState(c, gitTestPipelineRunName, timeout, PipelineRunSucceed(gitTestPipelineRunName), "PipelineRunCompleted"); err != nil { + taskruns, err := c.TaskRunClient.List(metav1.ListOptions{}) + if err != nil { + t.Errorf("Error getting TaskRun list for PipelineRun %s %s", gitTestPipelineRunName, err) + } + for _, tr := range taskruns.Items { + if tr.Status.PodName != "" { + p, err := c.KubeClient.Kube.CoreV1().Pods(namespace).Get(tr.Status.PodName, metav1.GetOptions{}) + if err != nil { + t.Fatalf("Error getting pod `%s` in namespace `%s`", tr.Status.PodName, namespace) + } + + for _, stat := range p.Status.ContainerStatuses { + if strings.HasPrefix(stat.Name, "step-git-source-"+gitSourceResourceName) { + if stat.State.Terminated != nil { + req := c.KubeClient.Kube.CoreV1().Pods(namespace).GetLogs(p.Name, &corev1.PodLogOptions{Container: stat.Name}) + logContent, err := req.Do().Raw() + if err != nil { + t.Fatalf("Error getting pod logs for pod `%s` and container `%s` in namespace `%s`", tr.Status.PodName, stat.Name, namespace) + } + // Check for failure messages from fetch and pull in the log file + if strings.Contains(strings.ToLower(string(logContent)), "could not resolve proxy: invalid.https.proxy.com") && + strings.Contains(strings.ToLower(string(logContent)), "pathspec 'master' did not match any file(s) known to git") { + t.Logf("Found exepected errors when using non-existent https proxy") + } else { + t.Logf("Container `%s` log File: %s", stat.Name, logContent) + t.Fatalf("The git code extraction did not fail as expected. Expected errors not found in log file.") + } + } + } + } + } + } + + } else { + t.Fatalf("PipelineRun succeeded when should have failed") + } +} + +func getGitPipelineResource(namespace, revision, sslverify, httpproxy, httpsproxy, noproxy string) *v1alpha1.PipelineResource { return tb.PipelineResource(gitSourceResourceName, namespace, tb.PipelineResourceSpec( v1alpha1.PipelineResourceTypeGit, tb.PipelineResourceSpecParam("Url", "https://github.com/tektoncd/pipeline"), tb.PipelineResourceSpecParam("Revision", revision), + tb.PipelineResourceSpecParam("sslVerify", sslverify), + tb.PipelineResourceSpecParam("httpProxy", httpproxy), + tb.PipelineResourceSpecParam("httpsProxy", httpsproxy), + tb.PipelineResourceSpecParam("noProxy", noproxy), )) }