Merge branch 'master' of github.com:meltwater/drone-cache into 206-do…

…ublestar-mount

CHANGELOG.md

@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 
 - [#209](https://github.com/meltwater/drone-cache/pull/209) Added double star directory searching in mounts (e.g. `path/**/subdir`)
+- [#198](https://github.com/meltwater/drone-cache/pull/198) Add `hashFiles` template function to generate the SHA256 hash of multiple files
 
 ### Changed
 

DOCS.md

@@ -33,6 +33,7 @@ Cache key template syntax is very basic. You just need to provide a string. In t
 Also following helper functions provided for your use:
 
 * `checksum`: Provides md5 hash of a file for given path
+* `hashFiles`: Provides SHA256 hash after SHA256 hashing each single file
 * `epoch`: Provides Unix epoch
 * `arch`: Provides Architecture of running system
 * `os`: Provides Operation system of running system
@@ -44,6 +45,10 @@ For further information about this syntax please see [official docs](https://gol
 `"{{ .Repo.Name }}-{{ .Commit.Branch }}-{{ checksum "go.mod" }}-yadayadayada"`
 
 `"{{ .Repo.Name }}_{{ checksum "go.mod" }}_{{ checksum "go.sum" }}_{{ arch }}_{{ os }}"`
+
+`"{{ .Repo.Name }}_{{ hashFiles "go.mod" "go.sum" }}_{{ arch }}_{{ os }}"`
+
+`"{{ .Repo.Name }}_{{ hashFiles "go.*" }}_{{ arch }}_{{ os }}"`
 *Metadata*
 
 Following metadata object is available and pre-populated with current build information for you to use in cache key templates.

README.md

@@ -70,7 +70,7 @@ steps:
     pull: true
     settings:
       restore: true
-      cache_key: {{ .Commit.Branch }}-{{ checksum "go.mod" }} # default if ommitted is {{ .Commit.Branch }}
+      cache_key: '{{ .Commit.Branch }}-{{ checksum "go.mod" }}' # default if ommitted is {{ .Commit.Branch }}
       bucket: drone-cache-bucket
       region: eu-west-1
       mount:
@@ -92,7 +92,7 @@ steps:
         from_secret: aws_secret_access_key
     settings:
       rebuild: true
-      cache_key: {{ .Commit.Branch }}-{{ checksum "go.mod" }} # default if ommitted is {{ .Commit.Branch }}
+      cache_key: '{{ .Commit.Branch }}-{{ checksum "go.mod" }}' # default if ommitted is {{ .Commit.Branch }}
       bucket: drone-cache-bucket
       region: eu-west-1
       mount:

docs/cache_key_templates.md

@@ -5,6 +5,7 @@ Cache key template syntax is very basic. You just need to provide a string. In t
 Also following helper functions provided for your use:
 
 * `checksum`: Provides md5 hash of a file for given path
+* `hashFiles`: Provides SHA256 hash after SHA256 hashing each single file
 * `epoch`: Provides Unix epoch
 * `arch`: Provides Architecture of running system
 * `os`: Provides Operation system of running system
@@ -17,6 +18,10 @@ For further information about this syntax please see [official docs](https://gol
 
 `"{{ .Repo.Name }}_{{ checksum "go.mod" }}_{{ checksum "go.sum" }}_{{ arch }}_{{ os }}"`
 
+`"{{ .Repo.Name }}_{{ hashFiles "go.mod" "go.sum" }}_{{ arch }}_{{ os }}"`
+
+`"{{ .Repo.Name }}_{{ hashFiles "go.*" }}_{{ arch }}_{{ os }}"`
+
 ## Metadata
 
 Following metadata object is available and pre-populated with current build information for you to use in cache key templates.

internal/plugin/plugin.go

@@ -2,10 +2,12 @@
 package plugin
 
 import (
+	"crypto/md5" // #nosec
 	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
+	"time"
 
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
@@ -83,14 +85,14 @@ func (p *Plugin) Exec() error { // nolint: funlen,cyclop
 
 	var generator key.Generator
 	if cfg.CacheKeyTemplate != "" {
-		generator = keygen.NewMetadata(p.logger, cfg.CacheKeyTemplate, p.Metadata)
+		generator = keygen.NewMetadata(p.logger, cfg.CacheKeyTemplate, p.Metadata, time.Now)
 		if err := generator.Check(); err != nil {
 			return fmt.Errorf("parse failed, falling back to default, %w", err)
 		}
 
-		options = append(options, cache.WithFallbackGenerator(keygen.NewHash(p.Metadata.Commit.Branch)))
+		options = append(options, cache.WithFallbackGenerator(keygen.NewHash(md5.New, p.Metadata.Commit.Branch)))
 	} else {
-		generator = keygen.NewHash(p.Metadata.Commit.Branch)
+		generator = keygen.NewHash(md5.New, p.Metadata.Commit.Branch)
 		options = append(options, cache.WithFallbackGenerator(keygen.NewStatic(p.Metadata.Commit.Branch)))
 	}
 

key/generator/hash.go

@@ -2,39 +2,41 @@ package generator
 
 import (
 	"fmt"
+	hash2 "hash"
 	"io"
 	"strings"
 )
 
-// Hash implements a key generator with md5.
+// Hash implements a key generator that uses the specified hash algorithm.
 type Hash struct {
+	hasher       func() hash2.Hash
 	defaultParts []string
 }
 
-// NewHash creates a new hash kye generator.
-func NewHash(defaultParts ...string) *Hash {
-	return &Hash{defaultParts: defaultParts}
+// NewHash creates a new hash key generator.
+func NewHash(hasher func() hash2.Hash, defaultParts ...string) *Hash {
+	return &Hash{
+		hasher:       hasher,
+		defaultParts: defaultParts,
+	}
 }
 
 // Generate generates key from given parts or templates as parameter.
 func (h *Hash) Generate(parts ...string) (string, error) {
-	key, err := hash(append(parts, h.defaultParts...)...)
+	parts = append(parts, h.defaultParts...)
+	readers := make([]io.Reader, len(parts))
+
+	for i, p := range parts {
+		readers[i] = strings.NewReader(p)
+	}
+
+	key, err := readerHasher(h.hasher, readers...)
 	if err != nil {
 		return "", fmt.Errorf("generate hash key for mounted, %w", err)
 	}
 
-	return key, nil
+	return fmt.Sprintf("%x", key), nil
 }
 
 // Check checks if generator functional.
 func (h *Hash) Check() error { return nil }
-
-// hash generates a key based on given strings (ie. filename paths and branch).
-func hash(parts ...string) (string, error) {
-	readers := make([]io.Reader, len(parts))
-	for i, p := range parts {
-		readers[i] = strings.NewReader(p)
-	}
-
-	return readerHasher(readers...)
-}

key/generator/hash_test.go

@@ -1,6 +1,7 @@
 package generator
 
 import (
+	"crypto/md5"
 	"testing"
 
 	"github.com/meltwater/drone-cache/test"
@@ -9,7 +10,7 @@ import (
 func TestGenerateHash(t *testing.T) {
 	t.Parallel()
 
-	actual, err := NewHash().Generate("hash")
+	actual, err := NewHash(md5.New).Generate("hash")
 	test.Ok(t, err)
 
 	expected := "0800fc577294c34e0b28ad2839435945"

key/generator/metadata.go

@@ -1,8 +1,13 @@
 package generator
 
 import (
+	"bytes"
+	"crypto/md5" // #nosec
+	"crypto/sha256"
 	"errors"
 	"fmt"
+	hash2 "hash"
+	"io"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -31,16 +36,17 @@ type Metadata struct {
 }
 
 // NewMetadata creates a new Key Generator.
-func NewMetadata(logger log.Logger, tmpl string, data metadata.Metadata) *Metadata {
+func NewMetadata(logger log.Logger, tmpl string, data metadata.Metadata, nowFunc func() time.Time) *Metadata {
 	return &Metadata{
 		logger: logger,
 		tmpl:   tmpl,
 		data:   data,
 		funcMap: template.FuncMap{
-			"checksum": checksumFunc(logger),
-			"epoch":    func() string { return strconv.FormatInt(time.Now().Unix(), EpochNumBase) },
-			"arch":     func() string { return runtime.GOARCH },
-			"os":       func() string { return runtime.GOOS },
+			"checksum":  checksumFunc(logger),
+			"hashFiles": hashFilesFunc(logger),
+			"epoch":     func() string { return strconv.FormatInt(nowFunc().Unix(), EpochNumBase) },
+			"arch":      func() string { return runtime.GOARCH },
+			"os":        func() string { return runtime.GOOS },
 		},
 	}
 }
@@ -89,29 +95,69 @@ func (g *Metadata) parseTemplate() (*template.Template, error) {
 
 func checksumFunc(logger log.Logger) func(string) string {
 	return func(p string) string {
-		path, err := filepath.Abs(filepath.Clean(p))
-		if err != nil {
-			level.Error(logger).Log("cache key template/checksum could not find file")
+		return fmt.Sprintf("%x", fileHash(p, logger, md5.New))
+	}
+}
 
-			return ""
+func hashFilesFunc(logger log.Logger) func(...string) string {
+	return func(patterns ...string) string {
+		var readers []io.Reader
+
+		for _, pattern := range patterns {
+			paths, err := filepath.Glob(pattern)
+			if err != nil {
+				level.Error(logger).Log("could not parse file path as a glob pattern")
+
+				continue
+			}
+
+			for _, p := range paths {
+				readers = append(readers, bytes.NewReader(fileHash(p, logger, sha256.New)))
+			}
 		}
 
-		f, err := os.Open(path)
-		if err != nil {
-			level.Error(logger).Log("cache key template/checksum could not open file")
+		if len(readers) == 0 {
+			level.Debug(logger).Log("no matches found for glob")
 
 			return ""
 		}
 
-		defer internal.CloseWithErrLogf(logger, f, "checksum close defer")
+		level.Debug(logger).Log("found %d files to hash", len(readers))
 
-		str, err := readerHasher(f)
+		h, err := readerHasher(sha256.New, readers...)
 		if err != nil {
-			level.Error(logger).Log("cache key template/checksum could not generate hash")
+			level.Error(logger).Log("could not generate the hash of the input files: %s", err.Error())
 
 			return ""
 		}
 
-		return str
+		return fmt.Sprintf("%x", h)
 	}
 }
+
+func fileHash(path string, logger log.Logger, hasher func() hash2.Hash) []byte {
+	path, err := filepath.Abs(filepath.Clean(path))
+	if err != nil {
+		level.Error(logger).Log("could not compute the absolute file path: %s", err.Error())
+
+		return []byte{}
+	}
+
+	f, err := os.Open(path)
+	if err != nil {
+		level.Error(logger).Log("could not open the file: %s", err.Error())
+
+		return []byte{}
+	}
+
+	defer internal.CloseWithErrLogf(logger, f, "checksum close defer")
+
+	h, err := readerHasher(hasher, f)
+	if err != nil {
+		level.Error(logger).Log("could not generate the hash of the input file: %s", err.Error())
+
+		return []byte{}
+	}
+
+	return h
+}

key/generator/metadata_test.go

@@ -1,8 +1,9 @@
 package generator
 
 import (
+	"runtime"
 	"testing"
-	"text/template"
+	"time"
 
 	"github.com/go-kit/log"
 	"github.com/meltwater/drone-cache/internal/metadata"
@@ -12,7 +13,7 @@ import (
 func TestGenerate(t *testing.T) {
 	t.Parallel()
 
-	l := log.NewNopLogger()
+	logger := log.NewNopLogger()
 
 	for _, tt := range []struct {
 		given    string
@@ -21,61 +22,28 @@ func TestGenerate(t *testing.T) {
 		{`{{ .Repo.Name }}`, "RepoName"},
 		{`{{ checksum "checksum_file_test.txt"}}`, "04a29c732ecbce101c1be44c948a50c6"},
 		{`{{ checksum "../../docs/drone_env_vars.md"}}`, "f8b5b7f96f3ffaa828e4890aab290e59"},
+		{`{{ hashFiles "" }}`, ""},
+		{`{{ hashFiles "checksum_file_test.txt" }}`, "b9fff559e00dd879bdffee979ee73e08c67dee2117da071083d3b833cbff7bc8"},
+		{`{{ hashFiles "checksum_file_test.txt" "checksum_file_test.txt" }}`, "fed16eb2e98f501968c74e261feb26a8776b2ae03b205ad7302f949e75ca455f"},
+		{`{{ hashFiles "checksum_file_tes*.txt" }}`, "b9fff559e00dd879bdffee979ee73e08c67dee2117da071083d3b833cbff7bc8"},
 		{`{{ epoch }}`, "1550563151"},
-		{`{{ arch }}`, "amd64"},
-		{`{{ os }}`, "darwin"},
+		{`{{ arch }}`, runtime.GOARCH},
+		{`{{ os }}`, runtime.GOOS},
 	} {
 		tt := tt
 		t.Run(tt.given, func(t *testing.T) {
-			g := Metadata{
-				logger: l,
-				tmpl:   tt.given,
-				data:   metadata.Metadata{Repo: metadata.Repo{Name: "RepoName"}},
-				funcMap: template.FuncMap{
-					"checksum": checksumFunc(l),
-					"epoch":    func() string { return "1550563151" },
-					"arch":     func() string { return "amd64" },
-					"os":       func() string { return "darwin" },
+			g := NewMetadata(
+				logger,
+				tt.given,
+				metadata.Metadata{Repo: metadata.Repo{Name: "RepoName"}},
+				func() time.Time {
+					return time.Unix(1550563151, 0)
 				},
-			}
+			)
 
 			actual, err := g.Generate(tt.given)
 			test.Ok(t, err)
-			test.Equals(t, actual, tt.expected)
-		})
-	}
-}
-
-func TestParseTemplate(t *testing.T) {
-	t.Parallel()
-
-	l := log.NewNopLogger()
-
-	for _, tt := range []struct {
-		given string
-	}{
-		{`{{ .Repo.Name }}`},
-		{`{{ checksum "checksum_file_test.txt"}}`},
-		{`{{ epoch }}`},
-		{`{{ arch }}`},
-		{`{{ os }}`},
-	} {
-		tt := tt
-		t.Run(tt.given, func(t *testing.T) {
-			g := Metadata{
-				logger: l,
-				tmpl:   tt.given,
-				data:   metadata.Metadata{Repo: metadata.Repo{Name: "RepoName"}},
-				funcMap: template.FuncMap{
-					"checksum": checksumFunc(l),
-					"epoch":    func() string { return "1550563151" },
-					"arch":     func() string { return "amd64" },
-					"os":       func() string { return "darwin" },
-				},
-			}
-
-			_, err := g.parseTemplate()
-			test.Ok(t, err)
+			test.Equals(t, tt.expected, actual)
 		})
 	}
 }