Only verify size if we can (#1080)

* Only verify size if we can

Sometimes Content-Length isn't set, and sometimes we don't know what it
should be. Do our best regardless.

* Update internal/verify/verify.go

Co-authored-by: Jason Hall <[email protected]>

Co-authored-by: Jason Hall <[email protected]>

internal/verify/verify.go

@@ -26,6 +26,9 @@ import (
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 )
 
+// SizeUnknown is a sentinel value to indicate that the expected size is not known.
+const SizeUnknown = -1
+
 type verifyReader struct {
 	inner             io.Reader
 	hasher            hash.Hash
@@ -38,7 +41,7 @@ func (vc *verifyReader) Read(b []byte) (int, error) {
 	n, err := vc.inner.Read(b)
 	vc.gotSize += int64(n)
 	if err == io.EOF {
-		if vc.gotSize != vc.wantSize {
+		if vc.wantSize != SizeUnknown && vc.gotSize != vc.wantSize {
 			return n, fmt.Errorf("error verifying size; got %d, want %d", vc.gotSize, vc.wantSize)
 		}
 		got := hex.EncodeToString(vc.hasher.Sum(make([]byte, 0, vc.hasher.Size())))
@@ -56,12 +59,18 @@ func (vc *verifyReader) Read(b []byte) (int, error) {
 // The reader will only be read up to size bytes, to prevent resource
 // exhaustion. If EOF is returned before size bytes are read, an error is
 // returned.
+//
+// A size of SizeUnknown (-1) indicates disables size verification when the size
+// is unknown ahead of time.
 func ReadCloser(r io.ReadCloser, size int64, h v1.Hash) (io.ReadCloser, error) {
 	w, err := v1.Hasher(h.Algorithm)
 	if err != nil {
 		return nil, err
 	}
-	r2 := io.LimitReader(io.TeeReader(r, w), size)
+	var r2 io.Reader = r
+	if size != SizeUnknown {
+		r2 = io.LimitReader(io.TeeReader(r, w), size)
+	}
 	return &and.ReadCloser{
 		Reader: &verifyReader{
 			inner:    r2,

pkg/v1/remote/descriptor.go

@@ -358,7 +358,7 @@ func (f *fetcher) headManifest(ref name.Reference, acceptable []types.MediaType)
 	}, nil
 }
 
-func (f *fetcher) fetchBlob(ctx context.Context, h v1.Hash) (io.ReadCloser, error) {
+func (f *fetcher) fetchBlob(ctx context.Context, size int64, h v1.Hash) (io.ReadCloser, error) {
 	u := f.url("blobs", h.String())
 	req, err := http.NewRequest(http.MethodGet, u.String(), nil)
 	if err != nil {
@@ -375,10 +375,15 @@ func (f *fetcher) fetchBlob(ctx context.Context, h v1.Hash) (io.ReadCloser, erro
 		return nil, err
 	}
 
-	// Verify up to the content-length header value.
-	size := resp.ContentLength
-	if size == -1 {
-		return nil, fmt.Errorf("GET %s: response did not include Content-Length header", u.String())
+	// Do whatever we can.
+	// If we have an expected size and Content-Length doesn't match, return an error.
+	// If we don't have an expected size and we do have a Content-Length, use Content-Length.
+	if hsize := resp.ContentLength; hsize != -1 {
+		if size == verify.SizeUnknown {
+			size = hsize
+		} else if hsize != size {
+			return nil, fmt.Errorf("GET %s: Content-Length header %d does not match expected size %d", u.String(), hsize, size)
+		}
 	}
 
 	return verify.ReadCloser(resp.Body, size, h)

pkg/v1/remote/image.go

@@ -100,7 +100,7 @@ func (r *remoteImage) RawConfigFile() ([]byte, error) {
 		return nil, err
 	}
 
-	body, err := r.fetchBlob(r.context, m.Config.Digest)
+	body, err := r.fetchBlob(r.context, m.Config.Size, m.Config.Digest)
 	if err != nil {
 		return nil, err
 	}

pkg/v1/remote/layer.go

@@ -18,6 +18,7 @@ import (
 	"io"
 
 	"github.com/google/go-containerregistry/internal/redact"
+	"github.com/google/go-containerregistry/internal/verify"
 	"github.com/google/go-containerregistry/pkg/name"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/partial"
@@ -34,7 +35,7 @@ type remoteLayer struct {
 func (rl *remoteLayer) Compressed() (io.ReadCloser, error) {
 	// We don't want to log binary layers -- this can break terminals.
 	ctx := redact.NewContext(rl.context, "omitting binary blobs from logs")
-	return rl.fetchBlob(ctx, rl.digest)
+	return rl.fetchBlob(ctx, verify.SizeUnknown, rl.digest)
 }
 
 // Compressed implements partial.CompressedLayer