Stabilize CI

meltwater · Apr 5, 2020 · 952c087 · 952c087
1 parent a1c0662
commit 952c087
Show file tree

Hide file tree

Showing 21 changed files with 112 additions and 109 deletions.
diff --git a/.drone.yml b/.drone.yml
@@ -41,7 +41,7 @@ steps:
 - name: test
   image: golang:1.14.1-alpine3.11
   commands:
-  - go test -mod=vendor -short -cover -tags=integration ./...
+  - go test -mod=vendor -short -cover -failfast -tags=integration ./...
   environment:
     CGO_ENABLED: 0
     TEST_S3_ENDPOINT: minio:9000
@@ -290,11 +290,11 @@ services:
   commands:
   - fake-gcs-server -public-host fakegcs -scheme http
 - name: sftp
-  image: atmoz/sftp
+  image: atmoz/sftp:alpine
   ports:
   - 22
   commands:
-  - /entrypoint foo:pass:::upload
+  - /entrypoint foo:pass:::sftp_test bar:pass:::plugin_test
 - name: azurite
   image: mcr.microsoft.com/azure-storage/azurite
   commands:

diff --git a/Makefile b/Makefile
@@ -97,7 +97,7 @@ container-push-dev: container-dev
 .PHONY: test
 test: $(GOTEST_BIN)
 	docker-compose up -d && sleep 1
-	-$(GOTEST_BIN) -failfast -race -short -tags=integration ./...
+	-$(GOTEST_BIN) -race -short -cover -failfast -tags=integration ./...
 	docker-compose down -v
 
 .PHONY: test-integration

diff --git a/archive/gzip/gzip.go b/archive/gzip/gzip.go
@@ -29,7 +29,7 @@ func New(logger log.Logger, root string, skipSymlinks bool, compressionLevel int
 func (a *Archive) Create(srcs []string, w io.Writer) (int64, error) {
 	gw, err := gzip.NewWriterLevel(w, a.compressionLevel)
 	if err != nil {
-		return 0, fmt.Errorf("create archive writer %w", err)
+		return 0, fmt.Errorf("create archive writer, %w", err)
 	}
 
 	defer internal.CloseWithErrLogf(a.logger, gw, "gzip writer")

diff --git a/archive/tar/tar.go b/archive/tar/tar.go
@@ -45,11 +45,11 @@ func (a *Archive) Create(srcs []string, w io.Writer) (int64, error) {
 	for _, src := range srcs {
 		_, err := os.Lstat(src)
 		if err != nil {
-			return written, fmt.Errorf("make sure file or directory readable <%s>: %v, %w", src, err, ErrSourceNotReachable)
+			return written, fmt.Errorf("make sure file or directory readable <%s>: %v,, %w", src, err, ErrSourceNotReachable)
 		}
 
 		if err := filepath.Walk(src, writeToArchive(tw, a.root, a.skipSymlinks, &written)); err != nil {
-			return written, fmt.Errorf("walk, add all files to archive %w", err)
+			return written, fmt.Errorf("walk, add all files to archive, %w", err)
 		}
 	}
 
@@ -70,7 +70,7 @@ func writeToArchive(tw *tar.Writer, root string, skipSymlinks bool, written *int
 		// Create header for Regular files and Directories
 		h, err := tar.FileInfoHeader(fi, fi.Name())
 		if err != nil {
-			return fmt.Errorf("create header for <%s> %w", path, err)
+			return fmt.Errorf("create header for <%s>, %w", path, err)
 		}
 
 		if fi.Mode()&os.ModeSymlink != 0 { // isSymbolic
@@ -80,19 +80,19 @@ func writeToArchive(tw *tar.Writer, root string, skipSymlinks bool, written *int
 
 			var err error
 			if h, err = createSymlinkHeader(fi, path); err != nil {
-				return fmt.Errorf("create header for symbolic link %w", err)
+				return fmt.Errorf("create header for symbolic link, %w", err)
 			}
 		}
 
 		name, err := relative(root, path)
 		if err != nil {
-			return fmt.Errorf("relative name <%s>: <%s> %w", path, root, err)
+			return fmt.Errorf("relative name <%s>: <%s>, %w", path, root, err)
 		}
 
 		h.Name = name
 
 		if err := tw.WriteHeader(h); err != nil {
-			return fmt.Errorf("write header for <%s> %w", path, err)
+			return fmt.Errorf("write header for <%s>, %w", path, err)
 		}
 
 		if !fi.Mode().IsRegular() {
@@ -101,7 +101,7 @@ func writeToArchive(tw *tar.Writer, root string, skipSymlinks bool, written *int
 
 		n, err := writeFileToArchive(tw, path)
 		if err != nil {
-			return fmt.Errorf("write file to archive %w", err)
+			return fmt.Errorf("write file to archive, %w", err)
 		}
 
 		*written += n
@@ -118,7 +118,7 @@ func relative(parent string, path string) (string, error) {
 
 	rel, err := filepath.Rel(parent, filepath.Dir(path))
 	if err != nil {
-		return "", fmt.Errorf("relative path <%s>, base <%s> %w", rel, name, err)
+		return "", fmt.Errorf("relative path <%s>, base <%s>, %w", rel, name, err)
 	}
 
 	// NOTICE: filepath.Rel puts "../" when given path is not under parent.
@@ -130,12 +130,12 @@ func relative(parent string, path string) (string, error) {
 func createSymlinkHeader(fi os.FileInfo, path string) (*tar.Header, error) {
 	lnk, err := os.Readlink(path)
 	if err != nil {
-		return nil, fmt.Errorf("read link <%s> %w", path, err)
+		return nil, fmt.Errorf("read link <%s>, %w", path, err)
 	}
 
 	h, err := tar.FileInfoHeader(fi, lnk)
 	if err != nil {
-		return nil, fmt.Errorf("create symlink header for <%s> %w", path, err)
+		return nil, fmt.Errorf("create symlink header for <%s>, %w", path, err)
 	}
 
 	return h, nil
@@ -144,14 +144,14 @@ func createSymlinkHeader(fi os.FileInfo, path string) (*tar.Header, error) {
 func writeFileToArchive(tw io.Writer, path string) (n int64, err error) {
 	f, err := os.Open(path)
 	if err != nil {
-		return 0, fmt.Errorf("open file <%s> %w", path, err)
+		return 0, fmt.Errorf("open file <%s>, %w", path, err)
 	}
 
 	defer internal.CloseWithErrCapturef(&err, f, "write file to archive <%s>", path)
 
 	written, err := io.Copy(tw, f)
 	if err != nil {
-		return written, fmt.Errorf("copy the file <%s> data to the tarball %w", path, err)
+		return written, fmt.Errorf("copy the file <%s> data to the tarball, %w", path, err)
 	}
 
 	return written, nil
@@ -171,7 +171,7 @@ func (a *Archive) Extract(dst string, r io.Reader) (int64, error) {
 		case err == io.EOF: // if no more files are found return
 			return written, nil
 		case err != nil: // return any other error
-			return written, fmt.Errorf("tar reader %v: %w", err, ErrArchiveNotReadable)
+			return written, fmt.Errorf("tar reader %v:, %w", err, ErrArchiveNotReadable)
 		case h == nil: // if the header is nil, skip it
 			continue
 		}
@@ -182,14 +182,14 @@ func (a *Archive) Extract(dst string, r io.Reader) (int64, error) {
 		} else {
 			name, err := relative(dst, h.Name)
 			if err != nil {
-				return 0, fmt.Errorf("relative name %w", err)
+				return 0, fmt.Errorf("relative name, %w", err)
 			}
 
 			target = filepath.Join(dst, name)
 		}
 
 		if err := os.MkdirAll(filepath.Dir(target), defaultDirPermission); err != nil {
-			return 0, fmt.Errorf("ensure directory <%s> %w", target, err)
+			return 0, fmt.Errorf("ensure directory <%s>, %w", target, err)
 		}
 
 		switch h.Typeflag {
@@ -204,19 +204,19 @@ func (a *Archive) Extract(dst string, r io.Reader) (int64, error) {
 			written += n
 
 			if err != nil {
-				return written, fmt.Errorf("extract regular file %w", err)
+				return written, fmt.Errorf("extract regular file, %w", err)
 			}
 
 			continue
 		case tar.TypeSymlink:
 			if err := extractSymlink(h, target); err != nil {
-				return written, fmt.Errorf("extract symbolic link %w", err)
+				return written, fmt.Errorf("extract symbolic link, %w", err)
 			}
 
 			continue
 		case tar.TypeLink:
 			if err := extractLink(h, target); err != nil {
-				return written, fmt.Errorf("extract link %w", err)
+				return written, fmt.Errorf("extract link, %w", err)
 			}
 
 			continue
@@ -230,7 +230,7 @@ func (a *Archive) Extract(dst string, r io.Reader) (int64, error) {
 
 func extractDir(h *tar.Header, target string) error {
 	if err := os.MkdirAll(target, os.FileMode(h.Mode)); err != nil {
-		return fmt.Errorf("create directory <%s> %w", target, err)
+		return fmt.Errorf("create directory <%s>, %w", target, err)
 	}
 
 	return nil
@@ -239,38 +239,38 @@ func extractDir(h *tar.Header, target string) error {
 func extractRegular(h *tar.Header, tr io.Reader, target string) (n int64, err error) {
 	f, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(h.Mode))
 	if err != nil {
-		return 0, fmt.Errorf("open extracted file for writing <%s> %w", target, err)
+		return 0, fmt.Errorf("open extracted file for writing <%s>, %w", target, err)
 	}
 
 	defer internal.CloseWithErrCapturef(&err, f, "extract regular <%s>", target)
 
 	written, err := io.Copy(f, tr)
 	if err != nil {
-		return written, fmt.Errorf("copy extracted file for writing <%s> %w", target, err)
+		return written, fmt.Errorf("copy extracted file for writing <%s>, %w", target, err)
 	}
 
 	return written, nil
 }
 
 func extractSymlink(h *tar.Header, target string) error {
 	if err := unlink(target); err != nil {
-		return fmt.Errorf("unlink <%s> %w", target, err)
+		return fmt.Errorf("unlink <%s>, %w", target, err)
 	}
 
 	if err := os.Symlink(h.Linkname, target); err != nil {
-		return fmt.Errorf("create symbolic link <%s> %w", target, err)
+		return fmt.Errorf("create symbolic link <%s>, %w", target, err)
 	}
 
 	return nil
 }
 
 func extractLink(h *tar.Header, target string) error {
 	if err := unlink(target); err != nil {
-		return fmt.Errorf("unlink <%s> %w", target, err)
+		return fmt.Errorf("unlink <%s>, %w", target, err)
 	}
 
 	if err := os.Link(h.Linkname, target); err != nil {
-		return fmt.Errorf("create hard link <%s> %w", h.Linkname, err)
+		return fmt.Errorf("create hard link <%s>, %w", h.Linkname, err)
 	}
 
 	return nil

diff --git a/cache/flusher.go b/cache/flusher.go
@@ -30,14 +30,14 @@ func (f flusher) Flush(srcs []string) error {
 
 		files, err := f.store.List(src)
 		if err != nil {
-			return fmt.Errorf("flusher list %w", err)
+			return fmt.Errorf("flusher list, %w", err)
 		}
 
 		for _, file := range files {
 			if f.dirty(file) {
 				err := f.store.Delete(file.Path)
 				if err != nil {
-					return fmt.Errorf("flusher delete %w", err)
+					return fmt.Errorf("flusher delete, %w", err)
 				}
 			}
 		}

diff --git a/cache/rebuilder.go b/cache/rebuilder.go
@@ -42,7 +42,7 @@ func (r rebuilder) Rebuild(srcs []string) error {
 
 	key, err := r.generateKey()
 	if err != nil {
-		return fmt.Errorf("generate key %w", err)
+		return fmt.Errorf("generate key, %w", err)
 	}
 
 	var (
@@ -53,7 +53,7 @@ func (r rebuilder) Rebuild(srcs []string) error {
 
 	for _, src := range srcs {
 		if _, err := os.Lstat(src); err != nil {
-			return fmt.Errorf("source <%s>, make sure file or directory exists and readable %w", src, err)
+			return fmt.Errorf("source <%s>, make sure file or directory exists and readable, %w", src, err)
 		}
 
 		dst := filepath.Join(namespace, key, src)
@@ -66,15 +66,15 @@ func (r rebuilder) Rebuild(srcs []string) error {
 			defer wg.Done()
 
 			if err := r.rebuild(src, dst); err != nil {
-				errs.Add(fmt.Errorf("upload from <%s> to <%s> %w", src, dst, err))
+				errs.Add(fmt.Errorf("upload from <%s> to <%s>, %w", src, dst, err))
 			}
 		}(dst, src)
 	}
 
 	wg.Wait()
 
 	if errs.Err() != nil {
-		return fmt.Errorf("rebuild failed %w", errs)
+		return fmt.Errorf("rebuild failed, %w", errs)
 	}
 
 	level.Info(r.logger).Log("msg", "cache built", "took", time.Since(now))
@@ -86,7 +86,7 @@ func (r rebuilder) Rebuild(srcs []string) error {
 func (r rebuilder) rebuild(src, dst string) (err error) {
 	src, err = filepath.Abs(filepath.Clean(src))
 	if err != nil {
-		return fmt.Errorf("clean source path %w", err)
+		return fmt.Errorf("clean source path, %w", err)
 	}
 
 	pr, pw := io.Pipe()
@@ -101,7 +101,7 @@ func (r rebuilder) rebuild(src, dst string) (err error) {
 
 		written, err := r.a.Create([]string{src}, pw)
 		if err != nil {
-			if err := pw.CloseWithError(fmt.Errorf("archive write, pipe writer failed %w", err)); err != nil {
+			if err := pw.CloseWithError(fmt.Errorf("archive write, pipe writer failed, %w", err)); err != nil {
 				level.Error(r.logger).Log("msg", "pw close", "err", err)
 			}
 		}
@@ -115,7 +115,7 @@ func (r rebuilder) rebuild(src, dst string) (err error) {
 	tr := io.TeeReader(pr, sw)
 
 	if err := r.s.Put(dst, tr); err != nil {
-		err = fmt.Errorf("upload file, pipe reader failed %w", err)
+		err = fmt.Errorf("upload file, pipe reader failed, %w", err)
 		if err := pr.CloseWithError(err); err != nil {
 			level.Error(r.logger).Log("msg", "pr close", "err", err)
 		}

diff --git a/cache/restorer.go b/cache/restorer.go
@@ -39,7 +39,7 @@ func (r restorer) Restore(dsts []string) error {
 
 	key, err := r.generateKey()
 	if err != nil {
-		return fmt.Errorf("generate key %w", err)
+		return fmt.Errorf("generate key, %w", err)
 	}
 
 	var (
@@ -59,15 +59,15 @@ func (r restorer) Restore(dsts []string) error {
 			defer wg.Done()
 
 			if err := r.restore(src, dst); err != nil {
-				errs.Add(fmt.Errorf("download from <%s> to <%s> %w", src, dst, err))
+				errs.Add(fmt.Errorf("download from <%s> to <%s>, %w", src, dst, err))
 			}
 		}(src, dst)
 	}
 
 	wg.Wait()
 
 	if errs.Err() != nil {
-		return fmt.Errorf("restore failed %w", errs)
+		return fmt.Errorf("restore failed, %w", errs)
 	}
 
 	level.Info(r.logger).Log("msg", "cache restored", "took", time.Since(now))
@@ -86,7 +86,7 @@ func (r restorer) restore(src, dst string) (err error) {
 		level.Info(r.logger).Log("msg", "downloading archived directory", "remote", src, "local", dst)
 
 		if err := r.s.Get(src, pw); err != nil {
-			if err := pw.CloseWithError(fmt.Errorf("get file from storage backend, pipe writer failed %w", err)); err != nil {
+			if err := pw.CloseWithError(fmt.Errorf("get file from storage backend, pipe writer failed, %w", err)); err != nil {
 				level.Error(r.logger).Log("msg", "pw close", "err", err)
 			}
 		}
@@ -96,7 +96,7 @@ func (r restorer) restore(src, dst string) (err error) {
 
 	written, err := r.a.Extract(dst, pr)
 	if err != nil {
-		err = fmt.Errorf("extract files from downloaded archive, pipe reader failed %w", err)
+		err = fmt.Errorf("extract files from downloaded archive, pipe reader failed, %w", err)
 		if err := pr.CloseWithError(err); err != nil {
 			level.Error(r.logger).Log("msg", "pr close", "err", err)
 		}

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -15,10 +15,13 @@ services:
       - 4443:4443
     command: -public-host localhost -scheme http
   sftp:
-    image: atmoz/sftp
+    image: atmoz/sftp:alpine
     ports:
     - "22:22"
-    command: foo:pass:::upload
+    volumes:
+        - ./testdata/sftp_test:/home/foo/sftp_test
+        - ./testdata/plugin_test:/home/bar/plugin_test
+    command: foo:pass:::sftp_test bar:pass:::plugin_test
   azurite:
     image: mcr.microsoft.com/azure-storage/azurite
     ports:

diff --git a/internal/io.go b/internal/io.go
@@ -21,7 +21,7 @@ func CloseWithErrLogf(logger log.Logger, closer io.Closer, format string, a ...i
 		logger = log.NewLogfmtLogger(os.Stderr)
 	}
 
-	level.Warn(logger).Log("msg", "detected close error", "err", fmt.Errorf(format+" %w", append(a, err)...))
+	level.Warn(logger).Log("msg", "detected close error", "err", fmt.Errorf(format+", %w", append(a, err)...))
 }
 
 // CloseWithErrCapturef runs function and on error return error by argument including the given error..
@@ -34,7 +34,7 @@ func CloseWithErrCapturef(err *error, closer io.Closer, format string, a ...inte
 
 		mErr := MultiError{}
 		mErr.Add(*err)
-		mErr.Add(fmt.Errorf(format+" %w", append(a, cErr)...))
+		mErr.Add(fmt.Errorf(format+", %w", append(a, cErr)...))
 		*err = mErr.Err()
 
 		return