Towards eliminating v1util

pkg/v1/hash.go

@@ -23,6 +23,8 @@ import (
 	"io"
 	"strconv"
 	"strings"
+
+	"github.com/google/go-containerregistry/pkg/v1/internal/and"
 )
 
 // Hash is an unqualified digest of some content, e.g. sha256:deadbeef
@@ -121,3 +123,40 @@ func SHA256(r io.Reader) (Hash, int64, error) {
 		Hex:       hex.EncodeToString(hasher.Sum(make([]byte, 0, hasher.Size()))),
 	}, n, nil
 }
+
+type verifyReader struct {
+	inner    io.Reader
+	hasher   hash.Hash
+	expected Hash
+}
+
+// Read implements io.Reader
+func (vc *verifyReader) Read(b []byte) (int, error) {
+	n, err := vc.inner.Read(b)
+	if err == io.EOF {
+		got := hex.EncodeToString(vc.hasher.Sum(make([]byte, 0, vc.hasher.Size())))
+		if want := vc.expected.Hex; got != want {
+			return n, fmt.Errorf("error verifying %s checksum; got %q, want %q",
+				vc.expected.Algorithm, got, want)
+		}
+	}
+	return n, err
+}
+
+// VerifyReadCloser wraps the given io.ReadCloser to verify that its contents match
+// the provided Hash before io.EOF is returned.
+func VerifyReadCloser(r io.ReadCloser, h Hash) (io.ReadCloser, error) {
+	w, err := Hasher(h.Algorithm)
+	if err != nil {
+		return nil, err
+	}
+	r2 := io.TeeReader(r, w)
+	return &and.ReadCloser{
+		Reader: &verifyReader{
+			inner:    r2,
+			hasher:   w,
+			expected: h,
+		},
+		CloseFunc: r.Close,
+	}, nil
+}

pkg/v1/hash_test.go

@@ -15,7 +15,9 @@
 package v1
 
 import (
+	"bytes"
 	"encoding/json"
+	"io/ioutil"
 	"strconv"
 	"strings"
 	"testing"
@@ -30,14 +32,14 @@ func TestGoodHashes(t *testing.T) {
 	for _, s := range good {
 		h, err := NewHash(s)
 		if err != nil {
-			t.Errorf("Unexpected error parsing hash: %v", err)
+			t.Error("Unexpected error parsing hash:", err)
 		}
 		if got, want := h.String(), s; got != want {
 			t.Errorf("String(); got %q, want %q", got, want)
 		}
 		bytes, err := json.Marshal(h)
 		if err != nil {
-			t.Errorf("Unexpected error json.Marshaling hash: %v", err)
+			t.Error("Unexpected error json.Marshaling hash:", err)
 		}
 		if got, want := string(bytes), strconv.Quote(h.String()); got != want {
 			t.Errorf("json.Marshal(); got %q, want %q", got, want)
@@ -62,7 +64,7 @@ func TestBadHashes(t *testing.T) {
 	for _, s := range bad {
 		h, err := NewHash(s)
 		if err == nil {
-			t.Errorf("Expected error, got: %v", h)
+			t.Error("Expected error, got:", h)
 		}
 	}
 }
@@ -71,7 +73,7 @@ func TestSHA256(t *testing.T) {
 	input := "asdf"
 	h, n, err := SHA256(strings.NewReader(input))
 	if err != nil {
-		t.Errorf("SHA256(asdf) = %v", err)
+		t.Error("SHA256(asdf) =", err)
 	}
 	if got, want := h.Algorithm, "sha256"; got != want {
 		t.Errorf("Algorithm; got %v, want %v", got, want)
@@ -90,10 +92,10 @@ func TestTextMarshalling(t *testing.T) {
 	foo := make(map[Hash]string)
 	b, err := json.Marshal(foo)
 	if err != nil {
-		t.Fatalf("could not marshal: %v", err)
+		t.Fatal("could not marshal:", err)
 	}
 	if err := json.Unmarshal(b, &foo); err != nil {
-		t.Errorf("could not unmarshal: %v", err)
+		t.Error("could not unmarshal:", err)
 	}
 
 	h := &Hash{
@@ -113,3 +115,48 @@ func TestTextMarshalling(t *testing.T) {
 		t.Errorf("mismatched hash: %s != %s", h, g)
 	}
 }
+
+func mustHash(s string, t *testing.T) Hash {
+	h, _, err := SHA256(strings.NewReader(s))
+	if err != nil {
+		t.Fatalf("SHA256(%s) = %v", s, err)
+	}
+	return h
+}
+
+func TestVerificationFailure(t *testing.T) {
+	want := "This is the input string."
+	buf := bytes.NewBufferString(want)
+
+	verified, err := VerifyReadCloser(ioutil.NopCloser(buf), mustHash("not the same", t))
+	if err != nil {
+		t.Fatal("VerifyReadCloser() =", err)
+	}
+	if b, err := ioutil.ReadAll(verified); err == nil {
+		t.Errorf("ReadAll() = %q; want verification error", string(b))
+	}
+}
+
+func TestVerification(t *testing.T) {
+	want := "This is the input string."
+	buf := bytes.NewBufferString(want)
+
+	verified, err := VerifyReadCloser(ioutil.NopCloser(buf), mustHash(want, t))
+	if err != nil {
+		t.Fatal("VerifyReadCloser() =", err)
+	}
+	if _, err := ioutil.ReadAll(verified); err != nil {
+		t.Error("ReadAll() =", err)
+	}
+}
+
+func TestBadHash(t *testing.T) {
+	h := Hash{
+		Algorithm: "fake256",
+		Hex:       "whatever",
+	}
+	_, err := VerifyReadCloser(ioutil.NopCloser(strings.NewReader("hi")), h)
+	if err == nil {
+		t.Errorf("VerifyReadCloser() = %v, wanted err", err)
+	}
+}

pkg/v1/v1util/and_closer.go → pkg/v1/internal/and/and_closer.go

@@ -1,4 +1,4 @@
-// Copyright 2018 Google LLC All Rights Reserved.
+// Copyright 2020 Google LLC All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,36 +12,36 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package v1util
+package and
 
 import (
 	"io"
 )
 
-// readAndCloser implements io.ReadCloser by reading from a particular io.Reader
+// ReadCloser implements io.ReadCloser by reading from a particular io.Reader
 // and then calling the provided "Close()" method.
-type readAndCloser struct {
+type ReadCloser struct {
 	io.Reader
 	CloseFunc func() error
 }
 
-var _ io.ReadCloser = (*readAndCloser)(nil)
+var _ io.ReadCloser = (*ReadCloser)(nil)
 
 // Close implements io.ReadCloser
-func (rac *readAndCloser) Close() error {
+func (rac *ReadCloser) Close() error {
 	return rac.CloseFunc()
 }
 
-// writeAndCloser implements io.WriteCloser by reading from a particular io.Writer
+// WriteCloser implements io.WriteCloser by reading from a particular io.Writer
 // and then calling the provided "Close()" method.
-type writeAndCloser struct {
+type WriteCloser struct {
 	io.Writer
 	CloseFunc func() error
 }
 
-var _ io.WriteCloser = (*writeAndCloser)(nil)
+var _ io.WriteCloser = (*WriteCloser)(nil)
 
 // Close implements io.WriteCloser
-func (wac *writeAndCloser) Close() error {
+func (wac *WriteCloser) Close() error {
 	return wac.CloseFunc()
 }

pkg/v1/v1util/and_closer_test.go → pkg/v1/internal/and/and_closer_test.go

@@ -1,4 +1,4 @@
-// Copyright 2018 Google LLC All Rights Reserved.
+// Copyright 2020 Google LLC All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package v1util
+package and
 
 import (
 	"bytes"
@@ -25,7 +25,7 @@ func TestRead(t *testing.T) {
 	r := bytes.NewBufferString(want)
 	called := false
 
-	rac := &readAndCloser{
+	rac := &ReadCloser{
 		Reader: r,
 		CloseFunc: func() error {
 			called = true
@@ -56,7 +56,7 @@ func TestWrite(t *testing.T) {
 	w := bytes.NewBuffer([]byte{})
 	called := false
 
-	wac := &writeAndCloser{
+	wac := &WriteCloser{
 		Writer: w,
 		CloseFunc: func() error {
 			called = true

pkg/v1/internal/gzip/zip.go

@@ -0,0 +1,96 @@
+// Copyright 2020 Google LLC All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package gzip
+
+import (
+	"bytes"
+	"compress/gzip"
+	"io"
+
+	"github.com/google/go-containerregistry/pkg/v1/internal/and"
+)
+
+var gzipMagicHeader = []byte{'\x1f', '\x8b'}
+
+// GzipReadCloser reads uncompressed input data from the io.ReadCloser and
+// returns an io.ReadCloser from which compressed data may be read.
+// This uses gzip.BestSpeed for the compression level.
+func GzipReadCloser(r io.ReadCloser) io.ReadCloser {
+	return GzipReadCloserLevel(r, gzip.BestSpeed)
+}
+
+// GzipReadCloserLevel reads uncompressed input data from the io.ReadCloser and
+// returns an io.ReadCloser from which compressed data may be read.
+// Refer to compress/gzip for the level:
+// https://golang.org/pkg/compress/gzip/#pkg-constants
+func GzipReadCloserLevel(r io.ReadCloser, level int) io.ReadCloser {
+	pr, pw := io.Pipe()
+
+	// Returns err so we can pw.CloseWithError(err)
+	go func() error {
+		// TODO(go1.14): Just defer {pw,gw,r}.Close like you'd expect.
+		// Context: https://golang.org/issue/24283
+		gw, err := gzip.NewWriterLevel(pw, level)
+		if err != nil {
+			return pw.CloseWithError(err)
+		}
+
+		if _, err := io.Copy(gw, r); err != nil {
+			defer r.Close()
+			defer gw.Close()
+			return pw.CloseWithError(err)
+		}
+		defer pw.Close()
+		defer r.Close()
+		defer gw.Close()
+
+		return nil
+	}()
+
+	return pr
+}
+
+// GunzipReadCloser reads compressed input data from the io.ReadCloser and
+// returns an io.ReadCloser from which uncompessed data may be read.
+func GunzipReadCloser(r io.ReadCloser) (io.ReadCloser, error) {
+	gr, err := gzip.NewReader(r)
+	if err != nil {
+		return nil, err
+	}
+	return &and.ReadCloser{
+		Reader: gr,
+		CloseFunc: func() error {
+			// If the unzip fails, then this seems to return the same
+			// error as the read.  We don't want this to interfere with
+			// us closing the main ReadCloser, since this could leave
+			// an open file descriptor (fails on Windows).
+			gr.Close()
+			return r.Close()
+		},
+	}, nil
+}
+
+// IsGzipped detects whether the input stream is compressed.
+func IsGzipped(r io.Reader) (bool, error) {
+	magicHeader := make([]byte, 2)
+	n, err := r.Read(magicHeader)
+	if n == 0 && err == io.EOF {
+		return false, nil
+	}
+	if err != nil {
+		return false, err
+	}
+	return bytes.Equal(magicHeader, gzipMagicHeader), nil
+}

pkg/v1/v1util/zip_test.go → pkg/v1/internal/gzip/zip_test.go

@@ -1,4 +1,4 @@
-// Copyright 2018 Google LLC All Rights Reserved.
+// Copyright 2020 Google LLC All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package v1util
+package gzip
 
 import (
 	"bytes"

pkg/v1/mutate/mutate.go

@@ -26,9 +26,9 @@ import (
 
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/empty"
+	"github.com/google/go-containerregistry/pkg/v1/internal/gzip"
 	"github.com/google/go-containerregistry/pkg/v1/tarball"
 	"github.com/google/go-containerregistry/pkg/v1/types"
-	"github.com/google/go-containerregistry/pkg/v1/v1util"
 )
 
 const whiteoutPrefix = ".wh."
@@ -335,7 +335,7 @@ func layerTime(layer v1.Layer, t time.Time) (v1.Layer, error) {
 	b := w.Bytes()
 	// gzip the contents, then create the layer
 	opener := func() (io.ReadCloser, error) {
-		return v1util.GzipReadCloser(ioutil.NopCloser(bytes.NewReader(b))), nil
+		return gzip.GzipReadCloser(ioutil.NopCloser(bytes.NewReader(b))), nil
 	}
 	layer, err = tarball.LayerFromOpener(opener)
 	if err != nil {

pkg/v1/partial/compressed.go

@@ -18,8 +18,8 @@ import (
 	"io"
 
 	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/internal/gzip"
 	"github.com/google/go-containerregistry/pkg/v1/types"
-	"github.com/google/go-containerregistry/pkg/v1/v1util"
 )
 
 // CompressedLayer represents the bare minimum interface a natively
@@ -49,7 +49,7 @@ func (cle *compressedLayerExtender) Uncompressed() (io.ReadCloser, error) {
 	if err != nil {
 		return nil, err
 	}
-	return v1util.GunzipReadCloser(r)
+	return gzip.GunzipReadCloser(r)
 }
 
 // DiffID implements v1.Layer