Initial implementation of go-git client (#2720)

Signed-off-by: Azeem Shaikh <[email protected]>

clients/git/client.go

@@ -0,0 +1,209 @@
+// Copyright 2021 OpenSSF Scorecard Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package git defines helper functions for clients.RepoClient interface.
+package git
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"sync"
+
+	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing"
+	cp "github.com/otiai10/copy"
+
+	"github.com/ossf/scorecard/v4/clients"
+)
+
+const repoDir = "repo*"
+
+var (
+	errNilCommitFound = errors.New("nil commit found")
+	errEmptyQuery     = errors.New("query is empty")
+)
+
+type Client struct {
+	gitRepo        *git.Repository
+	worktree       *git.Worktree
+	listCommits    *sync.Once
+	tempDir        string
+	errListCommits error
+	commits        []clients.Commit
+	commitDepth    int
+}
+
+func (c *Client) InitRepo(uri, commitSHA string, commitDepth int) error {
+	// cleanup previous state, if any.
+	c.Close()
+	c.listCommits = new(sync.Once)
+	c.commits = nil
+
+	// init
+	c.commitDepth = commitDepth
+	tempDir, err := os.MkdirTemp("", repoDir)
+	if err != nil {
+		return fmt.Errorf("os.MkdirTemp: %w", err)
+	}
+
+	// git clone
+	const filePrefix = "file://"
+	if strings.HasPrefix(uri, filePrefix) {
+		if err := cp.Copy(strings.TrimPrefix(uri, filePrefix), tempDir); err != nil {
+			return fmt.Errorf("cp.Copy: %w", err)
+		}
+		c.gitRepo, err = git.PlainOpen(tempDir)
+		if err != nil {
+			return fmt.Errorf("git.PlainOpen: %w", err)
+		}
+	} else {
+		c.gitRepo, err = git.PlainClone(tempDir, false /*isBare*/, &git.CloneOptions{
+			URL:      uri,
+			Progress: os.Stdout,
+		})
+		if err != nil {
+			return fmt.Errorf("git.PlainClone: %w", err)
+		}
+	}
+	c.tempDir = tempDir
+	c.worktree, err = c.gitRepo.Worktree()
+	if err != nil {
+		return fmt.Errorf("git.Worktree: %w", err)
+	}
+
+	// git checkout
+	if commitSHA != clients.HeadSHA {
+		if err := c.worktree.Checkout(&git.CheckoutOptions{
+			Hash:  plumbing.NewHash(commitSHA),
+			Force: true, // throw away any unsaved changes.
+		}); err != nil {
+			return fmt.Errorf("git.Worktree: %w", err)
+		}
+	}
+
+	return nil
+}
+
+func (c *Client) ListCommits() ([]clients.Commit, error) {
+	c.listCommits.Do(func() {
+		commitIter, err := c.gitRepo.Log(&git.LogOptions{
+			Order: git.LogOrderCommitterTime,
+		})
+		if err != nil {
+			c.errListCommits = fmt.Errorf("git.CommitObjects: %w", err)
+			return
+		}
+		c.commits = make([]clients.Commit, 0, c.commitDepth)
+		for i := 0; i < c.commitDepth; i++ {
+			commit, err := commitIter.Next()
+			if err != nil && !errors.Is(err, io.EOF) {
+				c.errListCommits = fmt.Errorf("commitIter.Next: %w", err)
+				return
+			}
+			// No more commits.
+			if errors.Is(err, io.EOF) {
+				break
+			}
+
+			if commit == nil {
+				// Not sure in what case a nil commit is returned. Fail explicitly.
+				c.errListCommits = fmt.Errorf("%w", errNilCommitFound)
+				return
+			}
+
+			c.commits = append(c.commits, clients.Commit{
+				SHA:           commit.Hash.String(),
+				Message:       commit.Message,
+				CommittedDate: commit.Committer.When,
+				Committer: clients.User{
+					Login: commit.Committer.Email,
+				},
+			})
+		}
+	})
+	return c.commits, c.errListCommits
+}
+
+func (c *Client) Search(request clients.SearchRequest) (clients.SearchResponse, error) {
+	// Pattern
+	if request.Query == "" {
+		return clients.SearchResponse{}, errEmptyQuery
+	}
+	queryRegexp, err := regexp.Compile(request.Query)
+	if err != nil {
+		return clients.SearchResponse{}, fmt.Errorf("regexp.Compile: %w", err)
+	}
+	grepOpts := &git.GrepOptions{
+		Patterns: []*regexp.Regexp{queryRegexp},
+	}
+
+	// path/filename
+	var pathExpr string
+	switch {
+	case request.Path != "" && request.Filename != "":
+		pathExpr = filepath.Join(fmt.Sprintf("^%s", request.Path),
+			fmt.Sprintf(".*%s$", request.Filename))
+	case request.Path != "":
+		pathExpr = fmt.Sprintf("^%s", request.Path)
+	case request.Filename != "":
+		pathExpr = filepath.Join(".*", fmt.Sprintf("%s$", request.Filename))
+	}
+	if pathExpr != "" {
+		pathRegexp, err := regexp.Compile(pathExpr)
+		if err != nil {
+			return clients.SearchResponse{}, fmt.Errorf("regexp.Compile: %w", err)
+		}
+		grepOpts.PathSpecs = append(grepOpts.PathSpecs, pathRegexp)
+	}
+
+	// Grep
+	grepResults, err := c.worktree.Grep(grepOpts)
+	if err != nil {
+		return clients.SearchResponse{}, fmt.Errorf("git.Grep: %w", err)
+	}
+
+	ret := clients.SearchResponse{}
+	for _, grepResult := range grepResults {
+		ret.Results = append(ret.Results, clients.SearchResult{
+			Path: grepResult.FileName,
+		})
+	}
+	ret.Hits = len(grepResults)
+	return ret, nil
+}
+
+// TODO(#1709): Implement below fns using go-git.
+func (c *Client) SearchCommits(request clients.SearchCommitsOptions) ([]clients.Commit, error) {
+	return nil, nil
+}
+
+func (c *Client) ListFiles(predicate func(string) (bool, error)) ([]string, error) {
+	return nil, nil
+}
+
+func (c *Client) GetFileContent(filename string) ([]byte, error) {
+	return nil, nil
+}
+
+func (c *Client) Close() error {
+	if err := os.RemoveAll(c.tempDir); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("os.RemoveAll: %w", err)
+	}
+	return nil
+}

clients/git/e2e_test.go

@@ -0,0 +1,155 @@
+// Copyright 2021 OpenSSF Scorecard Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package git
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"github.com/ossf/scorecard/v4/clients"
+)
+
+var _ = DescribeTable("Test ListCommits commit-depth for HEAD",
+	func(uri string) {
+		const commitSHA = clients.HeadSHA
+		const commitDepth = 1
+		client := &Client{}
+		Expect(client.InitRepo(uri, commitSHA, commitDepth)).To(BeNil())
+		commits, err := client.ListCommits()
+		Expect(err).To(BeNil())
+		Expect(len(commits)).Should(BeEquivalentTo(commitDepth))
+		Expect(client.Close()).To(BeNil())
+	},
+	Entry("GitHub", "https://github.com/ossf/scorecard"),
+	Entry("Local", "file://../../"),
+	Entry("GitLab", "https://gitlab.haskell.org/haskell/filepath"),
+)
+
+var _ = DescribeTable("Test ListCommits commit-depth and latest commit at [0]",
+	func(uri, commitSHA string) {
+		const commitDepth = 10
+		client := &Client{}
+		Expect(client.InitRepo(uri, commitSHA, commitDepth)).To(BeNil())
+		commits, err := client.ListCommits()
+		Expect(err).To(BeNil())
+		Expect(len(commits)).Should(BeEquivalentTo(commitDepth))
+		Expect(commits[0].SHA).Should(BeEquivalentTo(commitSHA))
+		Expect(client.Close()).To(BeNil())
+	},
+	Entry("GitHub", "https://github.com/ossf/scorecard", "c06ac740cc49fea404c54c036000731d5ea6ebe3"),
+	Entry("Local", "file://../../", "c06ac740cc49fea404c54c036000731d5ea6ebe3"),
+	Entry("GitLab", "https://gitlab.haskell.org/haskell/filepath", "98f8bba9eac8c7183143d290d319be7df76c258b"),
+)
+
+var _ = DescribeTable("Test ListCommits without enough commits",
+	func(uri string) {
+		const commitSHA = "dc1835b7ffe526969d65436b621e171e3386771e"
+		const commitDepth = 10
+		client := &Client{}
+		Expect(client.InitRepo(uri, commitSHA, commitDepth)).To(BeNil())
+		commits, err := client.ListCommits()
+		Expect(err).To(BeNil())
+		Expect(len(commits)).Should(BeEquivalentTo(3))
+		Expect(commits[0].SHA).Should(BeEquivalentTo(commitSHA))
+		Expect(client.Close()).To(BeNil())
+	},
+	Entry("GitHub", "https://github.com/ossf/scorecard"),
+	Entry("Local", "file://../../"),
+	// TODO(#1709): Add equivalent test for GitLab.
+)
+
+var _ = DescribeTable("Test Search across a repo",
+	func(uri string) {
+		const (
+			commitSHA   = "c06ac740cc49fea404c54c036000731d5ea6ebe3"
+			commitDepth = 10
+		)
+		client := &Client{}
+		Expect(client.InitRepo(uri, commitSHA, commitDepth)).To(BeNil())
+		resp, err := client.Search(clients.SearchRequest{
+			Query: "github/codeql-action/analyze",
+		})
+		Expect(err).To(BeNil())
+		Expect(resp.Hits).Should(BeNumerically(">=", 1))
+		Expect(client.Close()).To(BeNil())
+	},
+	Entry("GitHub", "https://github.com/ossf/scorecard"),
+	Entry("Local", "file://../../"),
+	// TODO(#1709): Add equivalent test for GitLab.
+)
+
+var _ = DescribeTable("Test Search within a path",
+	func(uri string) {
+		const (
+			commitSHA   = "c06ac740cc49fea404c54c036000731d5ea6ebe3"
+			commitDepth = 10
+		)
+		client := &Client{}
+		Expect(client.InitRepo(uri, commitSHA, commitDepth)).To(BeNil())
+		resp, err := client.Search(clients.SearchRequest{
+			Query: "github/codeql-action/analyze",
+			Path:  ".github/workflows",
+		})
+		Expect(err).To(BeNil())
+		Expect(resp.Hits).Should(BeEquivalentTo(1))
+		Expect(client.Close()).To(BeNil())
+	},
+	Entry("GitHub", "https://github.com/ossf/scorecard"),
+	Entry("Local", "file://../../"),
+	// TODO(#1709): Add equivalent test for GitLab.
+)
+
+var _ = DescribeTable("Test Search within a filename",
+	func(uri string) {
+		const (
+			commitSHA   = "c06ac740cc49fea404c54c036000731d5ea6ebe3"
+			commitDepth = 10
+		)
+		client := &Client{}
+		Expect(client.InitRepo(uri, commitSHA, commitDepth)).To(BeNil())
+		resp, err := client.Search(clients.SearchRequest{
+			Query:    "github/codeql-action/analyze",
+			Filename: "codeql-analysis.yml",
+		})
+		Expect(err).To(BeNil())
+		Expect(resp.Hits).Should(BeEquivalentTo(1))
+		Expect(client.Close()).To(BeNil())
+	},
+	Entry("GitHub", "https://github.com/ossf/scorecard"),
+	Entry("Local", "file://../../"),
+	// TODO(#1709): Add equivalent test for GitLab.
+)
+
+var _ = DescribeTable("Test Search within path and filename",
+	func(uri string) {
+		const (
+			commitSHA   = "c06ac740cc49fea404c54c036000731d5ea6ebe3"
+			commitDepth = 10
+		)
+		client := &Client{}
+		Expect(client.InitRepo(uri, commitSHA, commitDepth)).To(BeNil())
+		resp, err := client.Search(clients.SearchRequest{
+			Query:    "github/codeql-action/analyze",
+			Path:     ".github/workflows",
+			Filename: "codeql-analysis.yml",
+		})
+		Expect(err).To(BeNil())
+		Expect(resp.Hits).Should(BeEquivalentTo(1))
+		Expect(client.Close()).To(BeNil())
+	},
+	Entry("GitHub", "https://github.com/ossf/scorecard"),
+	Entry("Local", "file://../../"),
+	// TODO(#1709): Add equivalent test for GitLab.
+)