From 952c0877864262a7b126ae08a4226938b2620a60 Mon Sep 17 00:00:00 2001 From: Kemal Akkoyun Date: Sun, 5 Apr 2020 11:52:46 +0200 Subject: [PATCH] Stabilize CI --- .drone.yml | 6 +-- Makefile | 2 +- archive/gzip/gzip.go | 2 +- archive/tar/tar.go | 50 ++++++++++++------------ cache/flusher.go | 4 +- cache/rebuilder.go | 14 +++---- cache/restorer.go | 10 ++--- docker-compose.yml | 7 +++- internal/io.go | 4 +- internal/plugin/plugin.go | 6 +-- internal/plugin/plugin_test.go | 10 +++-- key/generator/hash.go | 2 +- key/generator/metadata.go | 4 +- key/generator/util.go | 2 +- storage/backend/azure/azure.go | 10 ++--- storage/backend/backend.go | 2 +- storage/backend/filesystem/filesystem.go | 27 ++++++------- storage/backend/gcs/gcs.go | 12 +++--- storage/backend/s3/s3.go | 6 +-- storage/backend/sftp/sftp.go | 39 +++++++++--------- storage/backend/sftp/sftp_test.go | 2 +- 21 files changed, 112 insertions(+), 109 deletions(-) diff --git a/.drone.yml b/.drone.yml index 48c5f0ba..b9757637 100644 --- a/.drone.yml +++ b/.drone.yml @@ -41,7 +41,7 @@ steps: - name: test image: golang:1.14.1-alpine3.11 commands: - - go test -mod=vendor -short -cover -tags=integration ./... + - go test -mod=vendor -short -cover -failfast -tags=integration ./... environment: CGO_ENABLED: 0 TEST_S3_ENDPOINT: minio:9000 @@ -290,11 +290,11 @@ services: commands: - fake-gcs-server -public-host fakegcs -scheme http - name: sftp - image: atmoz/sftp + image: atmoz/sftp:alpine ports: - 22 commands: - - /entrypoint foo:pass:::upload + - /entrypoint foo:pass:::sftp_test bar:pass:::plugin_test - name: azurite image: mcr.microsoft.com/azure-storage/azurite commands: diff --git a/Makefile b/Makefile index 22633b04..da268c4f 100644 --- a/Makefile +++ b/Makefile @@ -97,7 +97,7 @@ container-push-dev: container-dev .PHONY: test test: $(GOTEST_BIN) docker-compose up -d && sleep 1 - -$(GOTEST_BIN) -failfast -race -short -tags=integration ./... + -$(GOTEST_BIN) -race -short -cover -failfast -tags=integration ./... docker-compose down -v .PHONY: test-integration diff --git a/archive/gzip/gzip.go b/archive/gzip/gzip.go index 2f4fd251..2fc2628e 100644 --- a/archive/gzip/gzip.go +++ b/archive/gzip/gzip.go @@ -29,7 +29,7 @@ func New(logger log.Logger, root string, skipSymlinks bool, compressionLevel int func (a *Archive) Create(srcs []string, w io.Writer) (int64, error) { gw, err := gzip.NewWriterLevel(w, a.compressionLevel) if err != nil { - return 0, fmt.Errorf("create archive writer %w", err) + return 0, fmt.Errorf("create archive writer, %w", err) } defer internal.CloseWithErrLogf(a.logger, gw, "gzip writer") diff --git a/archive/tar/tar.go b/archive/tar/tar.go index 304f17ba..51b0076b 100644 --- a/archive/tar/tar.go +++ b/archive/tar/tar.go @@ -45,11 +45,11 @@ func (a *Archive) Create(srcs []string, w io.Writer) (int64, error) { for _, src := range srcs { _, err := os.Lstat(src) if err != nil { - return written, fmt.Errorf("make sure file or directory readable <%s>: %v, %w", src, err, ErrSourceNotReachable) + return written, fmt.Errorf("make sure file or directory readable <%s>: %v,, %w", src, err, ErrSourceNotReachable) } if err := filepath.Walk(src, writeToArchive(tw, a.root, a.skipSymlinks, &written)); err != nil { - return written, fmt.Errorf("walk, add all files to archive %w", err) + return written, fmt.Errorf("walk, add all files to archive, %w", err) } } @@ -70,7 +70,7 @@ func writeToArchive(tw *tar.Writer, root string, skipSymlinks bool, written *int // Create header for Regular files and Directories h, err := tar.FileInfoHeader(fi, fi.Name()) if err != nil { - return fmt.Errorf("create header for <%s> %w", path, err) + return fmt.Errorf("create header for <%s>, %w", path, err) } if fi.Mode()&os.ModeSymlink != 0 { // isSymbolic @@ -80,19 +80,19 @@ func writeToArchive(tw *tar.Writer, root string, skipSymlinks bool, written *int var err error if h, err = createSymlinkHeader(fi, path); err != nil { - return fmt.Errorf("create header for symbolic link %w", err) + return fmt.Errorf("create header for symbolic link, %w", err) } } name, err := relative(root, path) if err != nil { - return fmt.Errorf("relative name <%s>: <%s> %w", path, root, err) + return fmt.Errorf("relative name <%s>: <%s>, %w", path, root, err) } h.Name = name if err := tw.WriteHeader(h); err != nil { - return fmt.Errorf("write header for <%s> %w", path, err) + return fmt.Errorf("write header for <%s>, %w", path, err) } if !fi.Mode().IsRegular() { @@ -101,7 +101,7 @@ func writeToArchive(tw *tar.Writer, root string, skipSymlinks bool, written *int n, err := writeFileToArchive(tw, path) if err != nil { - return fmt.Errorf("write file to archive %w", err) + return fmt.Errorf("write file to archive, %w", err) } *written += n @@ -118,7 +118,7 @@ func relative(parent string, path string) (string, error) { rel, err := filepath.Rel(parent, filepath.Dir(path)) if err != nil { - return "", fmt.Errorf("relative path <%s>, base <%s> %w", rel, name, err) + return "", fmt.Errorf("relative path <%s>, base <%s>, %w", rel, name, err) } // NOTICE: filepath.Rel puts "../" when given path is not under parent. @@ -130,12 +130,12 @@ func relative(parent string, path string) (string, error) { func createSymlinkHeader(fi os.FileInfo, path string) (*tar.Header, error) { lnk, err := os.Readlink(path) if err != nil { - return nil, fmt.Errorf("read link <%s> %w", path, err) + return nil, fmt.Errorf("read link <%s>, %w", path, err) } h, err := tar.FileInfoHeader(fi, lnk) if err != nil { - return nil, fmt.Errorf("create symlink header for <%s> %w", path, err) + return nil, fmt.Errorf("create symlink header for <%s>, %w", path, err) } return h, nil @@ -144,14 +144,14 @@ func createSymlinkHeader(fi os.FileInfo, path string) (*tar.Header, error) { func writeFileToArchive(tw io.Writer, path string) (n int64, err error) { f, err := os.Open(path) if err != nil { - return 0, fmt.Errorf("open file <%s> %w", path, err) + return 0, fmt.Errorf("open file <%s>, %w", path, err) } defer internal.CloseWithErrCapturef(&err, f, "write file to archive <%s>", path) written, err := io.Copy(tw, f) if err != nil { - return written, fmt.Errorf("copy the file <%s> data to the tarball %w", path, err) + return written, fmt.Errorf("copy the file <%s> data to the tarball, %w", path, err) } return written, nil @@ -171,7 +171,7 @@ func (a *Archive) Extract(dst string, r io.Reader) (int64, error) { case err == io.EOF: // if no more files are found return return written, nil case err != nil: // return any other error - return written, fmt.Errorf("tar reader %v: %w", err, ErrArchiveNotReadable) + return written, fmt.Errorf("tar reader %v:, %w", err, ErrArchiveNotReadable) case h == nil: // if the header is nil, skip it continue } @@ -182,14 +182,14 @@ func (a *Archive) Extract(dst string, r io.Reader) (int64, error) { } else { name, err := relative(dst, h.Name) if err != nil { - return 0, fmt.Errorf("relative name %w", err) + return 0, fmt.Errorf("relative name, %w", err) } target = filepath.Join(dst, name) } if err := os.MkdirAll(filepath.Dir(target), defaultDirPermission); err != nil { - return 0, fmt.Errorf("ensure directory <%s> %w", target, err) + return 0, fmt.Errorf("ensure directory <%s>, %w", target, err) } switch h.Typeflag { @@ -204,19 +204,19 @@ func (a *Archive) Extract(dst string, r io.Reader) (int64, error) { written += n if err != nil { - return written, fmt.Errorf("extract regular file %w", err) + return written, fmt.Errorf("extract regular file, %w", err) } continue case tar.TypeSymlink: if err := extractSymlink(h, target); err != nil { - return written, fmt.Errorf("extract symbolic link %w", err) + return written, fmt.Errorf("extract symbolic link, %w", err) } continue case tar.TypeLink: if err := extractLink(h, target); err != nil { - return written, fmt.Errorf("extract link %w", err) + return written, fmt.Errorf("extract link, %w", err) } continue @@ -230,7 +230,7 @@ func (a *Archive) Extract(dst string, r io.Reader) (int64, error) { func extractDir(h *tar.Header, target string) error { if err := os.MkdirAll(target, os.FileMode(h.Mode)); err != nil { - return fmt.Errorf("create directory <%s> %w", target, err) + return fmt.Errorf("create directory <%s>, %w", target, err) } return nil @@ -239,14 +239,14 @@ func extractDir(h *tar.Header, target string) error { func extractRegular(h *tar.Header, tr io.Reader, target string) (n int64, err error) { f, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(h.Mode)) if err != nil { - return 0, fmt.Errorf("open extracted file for writing <%s> %w", target, err) + return 0, fmt.Errorf("open extracted file for writing <%s>, %w", target, err) } defer internal.CloseWithErrCapturef(&err, f, "extract regular <%s>", target) written, err := io.Copy(f, tr) if err != nil { - return written, fmt.Errorf("copy extracted file for writing <%s> %w", target, err) + return written, fmt.Errorf("copy extracted file for writing <%s>, %w", target, err) } return written, nil @@ -254,11 +254,11 @@ func extractRegular(h *tar.Header, tr io.Reader, target string) (n int64, err er func extractSymlink(h *tar.Header, target string) error { if err := unlink(target); err != nil { - return fmt.Errorf("unlink <%s> %w", target, err) + return fmt.Errorf("unlink <%s>, %w", target, err) } if err := os.Symlink(h.Linkname, target); err != nil { - return fmt.Errorf("create symbolic link <%s> %w", target, err) + return fmt.Errorf("create symbolic link <%s>, %w", target, err) } return nil @@ -266,11 +266,11 @@ func extractSymlink(h *tar.Header, target string) error { func extractLink(h *tar.Header, target string) error { if err := unlink(target); err != nil { - return fmt.Errorf("unlink <%s> %w", target, err) + return fmt.Errorf("unlink <%s>, %w", target, err) } if err := os.Link(h.Linkname, target); err != nil { - return fmt.Errorf("create hard link <%s> %w", h.Linkname, err) + return fmt.Errorf("create hard link <%s>, %w", h.Linkname, err) } return nil diff --git a/cache/flusher.go b/cache/flusher.go index 79875984..458c2e0c 100644 --- a/cache/flusher.go +++ b/cache/flusher.go @@ -30,14 +30,14 @@ func (f flusher) Flush(srcs []string) error { files, err := f.store.List(src) if err != nil { - return fmt.Errorf("flusher list %w", err) + return fmt.Errorf("flusher list, %w", err) } for _, file := range files { if f.dirty(file) { err := f.store.Delete(file.Path) if err != nil { - return fmt.Errorf("flusher delete %w", err) + return fmt.Errorf("flusher delete, %w", err) } } } diff --git a/cache/rebuilder.go b/cache/rebuilder.go index bce9f4b2..f531d96e 100644 --- a/cache/rebuilder.go +++ b/cache/rebuilder.go @@ -42,7 +42,7 @@ func (r rebuilder) Rebuild(srcs []string) error { key, err := r.generateKey() if err != nil { - return fmt.Errorf("generate key %w", err) + return fmt.Errorf("generate key, %w", err) } var ( @@ -53,7 +53,7 @@ func (r rebuilder) Rebuild(srcs []string) error { for _, src := range srcs { if _, err := os.Lstat(src); err != nil { - return fmt.Errorf("source <%s>, make sure file or directory exists and readable %w", src, err) + return fmt.Errorf("source <%s>, make sure file or directory exists and readable, %w", src, err) } dst := filepath.Join(namespace, key, src) @@ -66,7 +66,7 @@ func (r rebuilder) Rebuild(srcs []string) error { defer wg.Done() if err := r.rebuild(src, dst); err != nil { - errs.Add(fmt.Errorf("upload from <%s> to <%s> %w", src, dst, err)) + errs.Add(fmt.Errorf("upload from <%s> to <%s>, %w", src, dst, err)) } }(dst, src) } @@ -74,7 +74,7 @@ func (r rebuilder) Rebuild(srcs []string) error { wg.Wait() if errs.Err() != nil { - return fmt.Errorf("rebuild failed %w", errs) + return fmt.Errorf("rebuild failed, %w", errs) } level.Info(r.logger).Log("msg", "cache built", "took", time.Since(now)) @@ -86,7 +86,7 @@ func (r rebuilder) Rebuild(srcs []string) error { func (r rebuilder) rebuild(src, dst string) (err error) { src, err = filepath.Abs(filepath.Clean(src)) if err != nil { - return fmt.Errorf("clean source path %w", err) + return fmt.Errorf("clean source path, %w", err) } pr, pw := io.Pipe() @@ -101,7 +101,7 @@ func (r rebuilder) rebuild(src, dst string) (err error) { written, err := r.a.Create([]string{src}, pw) if err != nil { - if err := pw.CloseWithError(fmt.Errorf("archive write, pipe writer failed %w", err)); err != nil { + if err := pw.CloseWithError(fmt.Errorf("archive write, pipe writer failed, %w", err)); err != nil { level.Error(r.logger).Log("msg", "pw close", "err", err) } } @@ -115,7 +115,7 @@ func (r rebuilder) rebuild(src, dst string) (err error) { tr := io.TeeReader(pr, sw) if err := r.s.Put(dst, tr); err != nil { - err = fmt.Errorf("upload file, pipe reader failed %w", err) + err = fmt.Errorf("upload file, pipe reader failed, %w", err) if err := pr.CloseWithError(err); err != nil { level.Error(r.logger).Log("msg", "pr close", "err", err) } diff --git a/cache/restorer.go b/cache/restorer.go index e3bb3add..4895e1a0 100644 --- a/cache/restorer.go +++ b/cache/restorer.go @@ -39,7 +39,7 @@ func (r restorer) Restore(dsts []string) error { key, err := r.generateKey() if err != nil { - return fmt.Errorf("generate key %w", err) + return fmt.Errorf("generate key, %w", err) } var ( @@ -59,7 +59,7 @@ func (r restorer) Restore(dsts []string) error { defer wg.Done() if err := r.restore(src, dst); err != nil { - errs.Add(fmt.Errorf("download from <%s> to <%s> %w", src, dst, err)) + errs.Add(fmt.Errorf("download from <%s> to <%s>, %w", src, dst, err)) } }(src, dst) } @@ -67,7 +67,7 @@ func (r restorer) Restore(dsts []string) error { wg.Wait() if errs.Err() != nil { - return fmt.Errorf("restore failed %w", errs) + return fmt.Errorf("restore failed, %w", errs) } level.Info(r.logger).Log("msg", "cache restored", "took", time.Since(now)) @@ -86,7 +86,7 @@ func (r restorer) restore(src, dst string) (err error) { level.Info(r.logger).Log("msg", "downloading archived directory", "remote", src, "local", dst) if err := r.s.Get(src, pw); err != nil { - if err := pw.CloseWithError(fmt.Errorf("get file from storage backend, pipe writer failed %w", err)); err != nil { + if err := pw.CloseWithError(fmt.Errorf("get file from storage backend, pipe writer failed, %w", err)); err != nil { level.Error(r.logger).Log("msg", "pw close", "err", err) } } @@ -96,7 +96,7 @@ func (r restorer) restore(src, dst string) (err error) { written, err := r.a.Extract(dst, pr) if err != nil { - err = fmt.Errorf("extract files from downloaded archive, pipe reader failed %w", err) + err = fmt.Errorf("extract files from downloaded archive, pipe reader failed, %w", err) if err := pr.CloseWithError(err); err != nil { level.Error(r.logger).Log("msg", "pr close", "err", err) } diff --git a/docker-compose.yml b/docker-compose.yml index 91b2df02..901aab82 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,10 +15,13 @@ services: - 4443:4443 command: -public-host localhost -scheme http sftp: - image: atmoz/sftp + image: atmoz/sftp:alpine ports: - "22:22" - command: foo:pass:::upload + volumes: + - ./testdata/sftp_test:/home/foo/sftp_test + - ./testdata/plugin_test:/home/bar/plugin_test + command: foo:pass:::sftp_test bar:pass:::plugin_test azurite: image: mcr.microsoft.com/azure-storage/azurite ports: diff --git a/internal/io.go b/internal/io.go index 62f3e840..85d60707 100644 --- a/internal/io.go +++ b/internal/io.go @@ -21,7 +21,7 @@ func CloseWithErrLogf(logger log.Logger, closer io.Closer, format string, a ...i logger = log.NewLogfmtLogger(os.Stderr) } - level.Warn(logger).Log("msg", "detected close error", "err", fmt.Errorf(format+" %w", append(a, err)...)) + level.Warn(logger).Log("msg", "detected close error", "err", fmt.Errorf(format+", %w", append(a, err)...)) } // CloseWithErrCapturef runs function and on error return error by argument including the given error.. @@ -34,7 +34,7 @@ func CloseWithErrCapturef(err *error, closer io.Closer, format string, a ...inte mErr := MultiError{} mErr.Add(*err) - mErr.Add(fmt.Errorf(format+" %w", append(a, cErr)...)) + mErr.Add(fmt.Errorf(format+", %w", append(a, cErr)...)) *err = mErr.Err() return diff --git a/internal/plugin/plugin.go b/internal/plugin/plugin.go index 33c702f5..d58c10e2 100644 --- a/internal/plugin/plugin.go +++ b/internal/plugin/plugin.go @@ -69,7 +69,7 @@ func (p *Plugin) Exec() error { } else { workspace, err := os.Getwd() if err != nil { - return fmt.Errorf("get working directory %w", err) + return fmt.Errorf("get working directory, %w", err) } localRoot = workspace @@ -86,7 +86,7 @@ func (p *Plugin) Exec() error { if cfg.CacheKeyTemplate != "" { generator = keygen.NewMetadata(p.logger, cfg.CacheKeyTemplate, p.Metadata) if err := generator.Check(); err != nil { - return fmt.Errorf("parse failed, falling back to default %w", err) + return fmt.Errorf("parse failed, falling back to default, %w", err) } options = append(options, cache.WithFallbackGenerator(keygen.NewHash(p.Metadata.Commit.Branch))) @@ -105,7 +105,7 @@ func (p *Plugin) Exec() error { SFTP: cfg.SFTP, }) if err != nil { - return fmt.Errorf("initialize backend <%s> %w", cfg.Backend, err) + return fmt.Errorf("initialize backend <%s>, %w", cfg.Backend, err) } // 3. Initialize cache. diff --git a/internal/plugin/plugin_test.go b/internal/plugin/plugin_test.go index c7aade46..77d5c148 100644 --- a/internal/plugin/plugin_test.go +++ b/internal/plugin/plugin_test.go @@ -41,6 +41,7 @@ const ( testRootMoved = "testdata/moved" defaultStorageOperationTimeout = 5 * time.Second defaultPublicHost = "localhost:4443" + repoName = "drone-cache" ) var publicHost = getEnv("TEST_STORAGE_EMULATOR_HOST", defaultPublicHost) @@ -180,6 +181,7 @@ func TestPlugin(t *testing.T) { // Compare test.EqualDirs(t, restoreRoot, testRootMounted, paths) + time.Sleep(time.Second) }) } } @@ -235,7 +237,7 @@ func newPlugin(c *Config) Plugin { Metadata: metadata.Metadata{ Repo: metadata.Repo{ Branch: "master", - Name: "drone-cache", + Name: repoName, }, Commit: metadata.Commit{ Branch: "master", @@ -423,9 +425,9 @@ func setupSFTP(t *testing.T, c *Config, name string) { const ( defaultSFTPHost = "127.0.0.1" defaultSFTPPort = "22" - defaultUsername = "foo" + defaultUsername = "bar" defaultPassword = "pass" - defaultCacheRoot = "/upload" + defaultCacheRoot = "/plugin_test" ) var ( @@ -447,7 +449,7 @@ func setupSFTP(t *testing.T, c *Config, name string) { client, err := pkgsftp.NewClient(sshClient) test.Ok(t, err) - test.Ok(t, client.MkdirAll(cacheRoot)) + test.Ok(t, client.MkdirAll(filepath.Join(cacheRoot, repoName))) t.Cleanup(func() { client.RemoveDirectory(cacheRoot) }) c.Backend = backend.SFTP diff --git a/key/generator/hash.go b/key/generator/hash.go index caab155e..cbe2bdc2 100644 --- a/key/generator/hash.go +++ b/key/generator/hash.go @@ -20,7 +20,7 @@ func NewHash(defaultParts ...string) *Hash { func (h *Hash) Generate(parts ...string) (string, error) { key, err := hash(append(parts, h.defaultParts...)...) if err != nil { - return "", fmt.Errorf("generate hash key for mounted %w", err) + return "", fmt.Errorf("generate hash key for mounted, %w", err) } return key, nil diff --git a/key/generator/metadata.go b/key/generator/metadata.go index ed5f2ec2..a7a4fd88 100644 --- a/key/generator/metadata.go +++ b/key/generator/metadata.go @@ -53,14 +53,14 @@ func (g *Metadata) Generate(_ ...string) (string, error) { t, err := g.parseTemplate() if err != nil { - return "", fmt.Errorf("parse, <%s> as cache key template, falling back to default %w", g.tmpl, err) + return "", fmt.Errorf("parse, <%s> as cache key template, falling back to default, %w", g.tmpl, err) } var b strings.Builder err = t.Execute(&b, g.data) if err != nil { - return "", fmt.Errorf("build, <%s> as cache key, falling back to default %w", g.tmpl, err) + return "", fmt.Errorf("build, <%s> as cache key, falling back to default, %w", g.tmpl, err) } return b.String(), nil diff --git a/key/generator/util.go b/key/generator/util.go index 42592334..3b248872 100644 --- a/key/generator/util.go +++ b/key/generator/util.go @@ -13,7 +13,7 @@ func readerHasher(readers ...io.Reader) (string, error) { for _, r := range readers { if _, err := io.Copy(h, r); err != nil { - return "", fmt.Errorf("write reader as hash %w", err) + return "", fmt.Errorf("write reader as hash, %w", err) } } diff --git a/storage/backend/azure/azure.go b/storage/backend/azure/azure.go index 2be21b7e..8b255615 100644 --- a/storage/backend/azure/azure.go +++ b/storage/backend/azure/azure.go @@ -38,7 +38,7 @@ func New(l log.Logger, c Config) (*Backend, error) { // 2. Create a default request pipeline using your storage account name and account key. credential, err := azblob.NewSharedKeyCredential(c.AccountName, c.AccountKey) if err != nil { - return nil, fmt.Errorf("azure, invalid credentials %w", err) + return nil, fmt.Errorf("azure, invalid credentials, %w", err) } // 3. Azurite has different URL pattern than production Azure Blob Storage. @@ -64,7 +64,7 @@ func New(l log.Logger, c Config) (*Backend, error) { if err != nil { ret, ok := err.(azblob.StorageError) if !ok { - return nil, fmt.Errorf("azure, unexpected error %w", err) + return nil, fmt.Errorf("azure, unexpected error, %w", err) } if ret.ServiceCode() == "ContainerAlreadyExists" { @@ -86,7 +86,7 @@ func (b *Backend) Get(ctx context.Context, p string, w io.Writer) (err error) { resp, err := blobURL.Download(ctx, 0, azblob.CountToEnd, azblob.BlobAccessConditions{}, false) if err != nil { - errCh <- fmt.Errorf("get the object %w", err) + errCh <- fmt.Errorf("get the object, %w", err) return } @@ -95,7 +95,7 @@ func (b *Backend) Get(ctx context.Context, p string, w io.Writer) (err error) { _, err = io.Copy(w, rc) if err != nil { - errCh <- fmt.Errorf("copy the object %w", err) + errCh <- fmt.Errorf("copy the object, %w", err) } }() @@ -118,7 +118,7 @@ func (b *Backend) Put(ctx context.Context, p string, r io.Reader) error { MaxBuffers: defaultMaxBuffers, }, ); err != nil { - return fmt.Errorf("put the object %w", err) + return fmt.Errorf("put the object, %w", err) } return nil diff --git a/storage/backend/backend.go b/storage/backend/backend.go index d1401b02..0ce775fd 100644 --- a/storage/backend/backend.go +++ b/storage/backend/backend.go @@ -82,7 +82,7 @@ func FromConfig(l log.Logger, backedType string, cfg Config) (Backend, error) { } if err != nil { - return nil, fmt.Errorf("initialize backend %w", err) + return nil, fmt.Errorf("initialize backend, %w", err) } return b, nil diff --git a/storage/backend/filesystem/filesystem.go b/storage/backend/filesystem/filesystem.go index b9157723..faf865c2 100644 --- a/storage/backend/filesystem/filesystem.go +++ b/storage/backend/filesystem/filesystem.go @@ -31,7 +31,7 @@ func New(l log.Logger, c Config) (*Backend, error) { //nolint: TODO(kakkoyun): Should it be created? if _, err := os.Stat(c.CacheRoot); err != nil { - return nil, fmt.Errorf("make sure volume is mounted, <%s> as cache root %w", c.CacheRoot, err) + return nil, fmt.Errorf("make sure volume is mounted, <%s> as cache root, %w", c.CacheRoot, err) } level.Debug(l).Log("msg", "Filesystem backend", "config", fmt.Sprintf("%#v", c)) @@ -43,7 +43,7 @@ func New(l log.Logger, c Config) (*Backend, error) { func (b *Backend) Get(ctx context.Context, p string, w io.Writer) error { path, err := filepath.Abs(filepath.Clean(filepath.Join(b.cacheRoot, p))) if err != nil { - return fmt.Errorf("absolute path %w", err) + return fmt.Errorf("absolute path, %w", err) } errCh := make(chan error) @@ -53,7 +53,7 @@ func (b *Backend) Get(ctx context.Context, p string, w io.Writer) error { rc, err := os.Open(path) if err != nil { - errCh <- fmt.Errorf("get the object %w", err) + errCh <- fmt.Errorf("get the object, %w", err) return } @@ -61,7 +61,7 @@ func (b *Backend) Get(ctx context.Context, p string, w io.Writer) error { _, err = io.Copy(w, rc) if err != nil { - errCh <- fmt.Errorf("copy the object %w", err) + errCh <- fmt.Errorf("copy the object, %w", err) } }() @@ -77,7 +77,12 @@ func (b *Backend) Get(ctx context.Context, p string, w io.Writer) error { func (b *Backend) Put(ctx context.Context, p string, r io.Reader) error { path, err := filepath.Abs(filepath.Clean(filepath.Join(b.cacheRoot, p))) if err != nil { - return fmt.Errorf("build path %w", err) + return fmt.Errorf("build path, %w", err) + } + + dir := filepath.Dir(path) + if err := os.MkdirAll(dir, os.FileMode(defaultFileMode)); err != nil { + return fmt.Errorf("create directory, %w", err) } errCh := make(chan error) @@ -85,26 +90,20 @@ func (b *Backend) Put(ctx context.Context, p string, r io.Reader) error { go func() { defer close(errCh) - dir := filepath.Dir(path) - if err := os.MkdirAll(dir, os.FileMode(defaultFileMode)); err != nil { - errCh <- fmt.Errorf("create directory %w", err) - return - } - w, err := os.Create(path) if err != nil { - errCh <- fmt.Errorf("create cache file %w", err) + errCh <- fmt.Errorf("create cache file, %w", err) return } defer internal.CloseWithErrLogf(b.logger, w, "file writer, close defer") if _, err := io.Copy(w, r); err != nil { - errCh <- fmt.Errorf("write contents of reader to a file %w", err) + errCh <- fmt.Errorf("write contents of reader to a file, %w", err) } if err := w.Close(); err != nil { - errCh <- fmt.Errorf("close the object %w", err) + errCh <- fmt.Errorf("close the object, %w", err) } }() diff --git a/storage/backend/gcs/gcs.go b/storage/backend/gcs/gcs.go index 092a11a1..cc2ebeed 100644 --- a/storage/backend/gcs/gcs.go +++ b/storage/backend/gcs/gcs.go @@ -51,7 +51,7 @@ func New(l log.Logger, c Config) (*Backend, error) { client, err := gcstorage.NewClient(ctx, opts...) if err != nil { - return nil, fmt.Errorf("gcs client initialization %w", err) + return nil, fmt.Errorf("gcs client initialization, %w", err) } return &Backend{ @@ -79,7 +79,7 @@ func (b *Backend) Get(ctx context.Context, p string, w io.Writer) error { r, err := obj.NewReader(ctx) if err != nil { - errCh <- fmt.Errorf("get the object %w", err) + errCh <- fmt.Errorf("get the object, %w", err) return } @@ -87,7 +87,7 @@ func (b *Backend) Get(ctx context.Context, p string, w io.Writer) error { _, err = io.Copy(w, r) if err != nil { - errCh <- fmt.Errorf("copy the object %w", err) + errCh <- fmt.Errorf("copy the object, %w", err) } }() @@ -118,16 +118,16 @@ func (b *Backend) Put(ctx context.Context, p string, r io.Reader) error { _, err := io.Copy(w, r) if err != nil { - errCh <- fmt.Errorf("copy the object %w", err) + errCh <- fmt.Errorf("copy the object, %w", err) } if err := w.Close(); err != nil { - errCh <- fmt.Errorf("close the object %w", err) + errCh <- fmt.Errorf("close the object, %w", err) } if b.acl != "" { if err := obj.ACL().Set(ctx, gcstorage.AllAuthenticatedUsers, gcstorage.ACLRole(b.acl)); err != nil { - errCh <- fmt.Errorf("set ACL of the object %w", err) + errCh <- fmt.Errorf("set ACL of the object, %w", err) } } }() diff --git a/storage/backend/s3/s3.go b/storage/backend/s3/s3.go index ec397189..061c08e0 100644 --- a/storage/backend/s3/s3.go +++ b/storage/backend/s3/s3.go @@ -72,7 +72,7 @@ func (b *Backend) Get(ctx context.Context, p string, w io.Writer) error { out, err := b.client.GetObjectWithContext(ctx, in) if err != nil { - errCh <- fmt.Errorf("get the object %w", err) + errCh <- fmt.Errorf("get the object, %w", err) return } @@ -80,7 +80,7 @@ func (b *Backend) Get(ctx context.Context, p string, w io.Writer) error { _, err = io.Copy(w, out.Body) if err != nil { - errCh <- fmt.Errorf("copy the object %w", err) + errCh <- fmt.Errorf("copy the object, %w", err) } }() @@ -109,7 +109,7 @@ func (b *Backend) Put(ctx context.Context, p string, r io.Reader) error { } if _, err := uploader.UploadWithContext(ctx, in); err != nil { - return fmt.Errorf("put the object %w", err) + return fmt.Errorf("put the object, %w", err) } return nil diff --git a/storage/backend/sftp/sftp.go b/storage/backend/sftp/sftp.go index 44cbad49..922facc8 100644 --- a/storage/backend/sftp/sftp.go +++ b/storage/backend/sftp/sftp.go @@ -28,7 +28,7 @@ type Backend struct { func New(l log.Logger, c Config) (*Backend, error) { authMethod, err := authMethod(c) if err != nil { - return nil, fmt.Errorf("unable to get ssh auth method %w", err) + return nil, fmt.Errorf("unable to get ssh auth method, %w", err) } /* #nosec */ @@ -39,18 +39,18 @@ func New(l log.Logger, c Config) (*Backend, error) { Timeout: c.Timeout, }) if err != nil { - return nil, fmt.Errorf("unable to connect to ssh %w", err) + return nil, fmt.Errorf("unable to connect to ssh, %w", err) } client, err := sftp.NewClient(sshClient) if err != nil { sshClient.Close() - return nil, fmt.Errorf("unable to connect to ssh with sftp protocol %w", err) + return nil, fmt.Errorf("unable to connect to ssh with sftp protocol, %w", err) } //nolint: TODO(kakkoyun): Should it be created? if _, err := client.Stat(c.CacheRoot); err != nil { - return nil, fmt.Errorf("make sure cache root <%s> created, %w", c.CacheRoot, err) + return nil, fmt.Errorf("make sure cache root <%s> created, %w", c.CacheRoot, err) } level.Debug(l).Log("msg", "sftp backend", "config", fmt.Sprintf("%#v", c)) @@ -62,7 +62,7 @@ func New(l log.Logger, c Config) (*Backend, error) { func (b *Backend) Get(ctx context.Context, p string, w io.Writer) error { path, err := filepath.Abs(filepath.Clean(filepath.Join(b.cacheRoot, p))) if err != nil { - return fmt.Errorf("generate absolute path %w", err) + return fmt.Errorf("generate absolute path, %w", err) } errCh := make(chan error) @@ -72,7 +72,7 @@ func (b *Backend) Get(ctx context.Context, p string, w io.Writer) error { rc, err := b.client.Open(path) if err != nil { - errCh <- fmt.Errorf("get the object %w", err) + errCh <- fmt.Errorf("get the object, %w", err) return } @@ -80,7 +80,7 @@ func (b *Backend) Get(ctx context.Context, p string, w io.Writer) error { _, err = io.Copy(w, rc) if err != nil { - errCh <- fmt.Errorf("copy the object %w", err) + errCh <- fmt.Errorf("copy the object, %w", err) } }() @@ -96,31 +96,30 @@ func (b *Backend) Get(ctx context.Context, p string, w io.Writer) error { func (b *Backend) Put(ctx context.Context, p string, r io.Reader) error { errCh := make(chan error) - go func() { - defer close(errCh) + path := filepath.Clean(filepath.Join(b.cacheRoot, p)) - path := filepath.Clean(filepath.Join(b.cacheRoot, p)) + dir := filepath.Dir(path) + if err := b.client.MkdirAll(dir); err != nil { + return fmt.Errorf("create directory, %w", err) + } - dir := filepath.Dir(path) - if err := b.client.MkdirAll(dir); err != nil { - errCh <- fmt.Errorf("create directory %w", err) - return - } + go func() { + defer close(errCh) w, err := b.client.Create(path) if err != nil { - errCh <- fmt.Errorf("create cache file %w", err) + errCh <- fmt.Errorf("create cache file, %w", err) return } defer internal.CloseWithErrLogf(b.logger, w, "writer close defer") if _, err := io.Copy(w, r); err != nil { - errCh <- fmt.Errorf("write contents of reader to a file %w", err) + errCh <- fmt.Errorf("write contents of reader to a file, %w", err) } if err := w.Close(); err != nil { - errCh <- fmt.Errorf("close the object %w", err) + errCh <- fmt.Errorf("close the object, %w", err) } }() @@ -149,12 +148,12 @@ func authMethod(c Config) ([]ssh.AuthMethod, error) { func readPublicKeyFile(file string) (ssh.AuthMethod, error) { buffer, err := ioutil.ReadFile(file) if err != nil { - return nil, fmt.Errorf("unable to read file %w", err) + return nil, fmt.Errorf("unable to read file, %w", err) } key, err := ssh.ParsePrivateKey(buffer) if err != nil { - return nil, fmt.Errorf("unable to parse private key %w", err) + return nil, fmt.Errorf("unable to parse private key, %w", err) } return ssh.PublicKeys(key), nil diff --git a/storage/backend/sftp/sftp_test.go b/storage/backend/sftp/sftp_test.go index 51f3ad6e..0f80bfc1 100644 --- a/storage/backend/sftp/sftp_test.go +++ b/storage/backend/sftp/sftp_test.go @@ -20,7 +20,7 @@ const ( defaultSFTPPort = "22" defaultUsername = "foo" defaultPassword = "pass" - defaultCacheRoot = "/upload" + defaultCacheRoot = "/sftp_test" ) var (