copy: add --dest-compress-format and --dest-compress-level

add the possibility to specify the format and the level to use when
compressing blobs.

Signed-off-by: Giuseppe Scrivano <[email protected]>

cmd/skopeo/utils.go

@@ -6,6 +6,7 @@ import (
 	"io"
 	"strings"
 
+	"github.com/containers/image/pkg/compression"
 	"github.com/containers/image/transports/alltransports"
 	"github.com/containers/image/types"
 	"github.com/urfave/cli"
@@ -147,15 +148,23 @@ func (opts *imageOptions) newSystemContext() (*types.SystemContext, error) {
 	if opts.noCreds {
 		ctx.DockerAuthConfig = &types.DockerAuthConfig{}
 	}
+	cf, err := compression.AlgorithmByName("gzip")
+	if err != nil {
+		return nil, err
+	}
+	ctx.CompressionFormat = &cf
+
 	return ctx, nil
 }
 
 // imageDestOptions is a superset of imageOptions specialized for iamge destinations.
 type imageDestOptions struct {
 	*imageOptions
-	osTreeTmpDir                string // A directory to use for OSTree temporary files
-	dirForceCompression         bool   // Compress layers when saving to the dir: transport
-	ociAcceptUncompressedLayers bool   // Whether to accept uncompressed layers in the oci: transport
+	osTreeTmpDir                string      // A directory to use for OSTree temporary files
+	dirForceCompression         bool        // Compress layers when saving to the dir: transport
+	ociAcceptUncompressedLayers bool        // Whether to accept uncompressed layers in the oci: transport
+	compressionFormat           string      // Format to use for the compression
+	compressionLevel            optionalInt // Level to use for the compression
 }
 
 // imageDestFlags prepares a collection of CLI flags writing into imageDestOptions, and the managed imageDestOptions structure.
@@ -179,6 +188,17 @@ func imageDestFlags(global *globalOptions, shared *sharedImageOptions, flagPrefi
 			Usage:       "Allow uncompressed image layers when saving to an OCI image using the 'oci' transport. (default is to compress things that aren't compressed)",
 			Destination: &opts.ociAcceptUncompressedLayers,
 		},
+		cli.StringFlag{
+			Name:        flagPrefix + "compress-format",
+			Usage:       "`FORMAT` to use for the compression",
+			Destination: &opts.compressionFormat,
+			Value:       "gzip",
+		},
+		cli.GenericFlag{
+			Name:  flagPrefix + "compress-level",
+			Usage: "`LEVEL` to use for the compression",
+			Value: newOptionalIntValue(&opts.compressionLevel),
+		},
 	}...), &opts
 }
 
@@ -193,6 +213,14 @@ func (opts *imageDestOptions) newSystemContext() (*types.SystemContext, error) {
 	ctx.OSTreeTmpDirPath = opts.osTreeTmpDir
 	ctx.DirForceCompress = opts.dirForceCompression
 	ctx.OCIAcceptUncompressedLayers = opts.ociAcceptUncompressedLayers
+	cf, err := compression.AlgorithmByName(opts.compressionFormat)
+	if err != nil {
+		return nil, err
+	}
+	ctx.CompressionFormat = &cf
+	if opts.compressionLevel.present {
+		ctx.CompressionLevel = &opts.compressionLevel.value
+	}
 	return ctx, err
 }
 

cmd/skopeo/utils_test.go

@@ -45,6 +45,8 @@ func TestImageOptionsNewSystemContext(t *testing.T) {
 	opts := fakeImageOptions(t, "dest-", []string{}, []string{})
 	res, err := opts.newSystemContext()
 	require.NoError(t, err)
+
+	res.CompressionFormat = nil // reflect doesn't work well with functions
 	assert.Equal(t, &types.SystemContext{}, res)
 
 	// Set everything to non-default values.
@@ -62,6 +64,7 @@ func TestImageOptionsNewSystemContext(t *testing.T) {
 	})
 	res, err = opts.newSystemContext()
 	require.NoError(t, err)
+	res.CompressionFormat = nil // reflect doesn't work well with functions
 	assert.Equal(t, &types.SystemContext{
 		RegistriesDirPath:                 "/srv/registries.d",
 		AuthFilePath:                      "/srv/authfile",
@@ -74,6 +77,7 @@ func TestImageOptionsNewSystemContext(t *testing.T) {
 		DockerDaemonCertPath:              "/srv/cert-dir",
 		DockerDaemonHost:                  "daemon-host.example.com",
 		DockerDaemonInsecureSkipTLSVerify: true,
+		CompressionFormat:                 nil,
 	}, res)
 
 	// Global/per-command tlsVerify behavior
@@ -134,6 +138,8 @@ func TestImageDestOptionsNewSystemContext(t *testing.T) {
 	opts := fakeImageDestOptions(t, "dest-", []string{}, []string{})
 	res, err := opts.newSystemContext()
 	require.NoError(t, err)
+
+	res.CompressionFormat = nil // reflect doesn't work well with functions
 	assert.Equal(t, &types.SystemContext{}, res)
 
 	// Explicitly set everything to default, except for when the default is “not present”
@@ -142,6 +148,7 @@ func TestImageDestOptionsNewSystemContext(t *testing.T) {
 	})
 	res, err = opts.newSystemContext()
 	require.NoError(t, err)
+	res.CompressionFormat = nil // reflect doesn't work well with functions
 	assert.Equal(t, &types.SystemContext{}, res)
 
 	// Set everything to non-default values.
@@ -161,6 +168,7 @@ func TestImageDestOptionsNewSystemContext(t *testing.T) {
 	})
 	res, err = opts.newSystemContext()
 	require.NoError(t, err)
+	res.CompressionFormat = nil // reflect doesn't work well with functions
 	assert.Equal(t, &types.SystemContext{
 		RegistriesDirPath:                 "/srv/registries.d",
 		AuthFilePath:                      "/srv/authfile",
@@ -175,6 +183,7 @@ func TestImageDestOptionsNewSystemContext(t *testing.T) {
 		DockerDaemonHost:                  "daemon-host.example.com",
 		DockerDaemonInsecureSkipTLSVerify: true,
 		DirForceCompress:                  true,
+		CompressionFormat:                 nil,
 	}, res)
 
 	// Invalid option values in imageOptions

docs/skopeo-copy.1.md

@@ -58,6 +58,10 @@ If the authorization state is not found there, $HOME/.docker/config.json is chec
 
 Existing signatures, if any, are preserved as well.
 
+**--dest-compress-format** _format_ Specifies the compression format to use.
+
+**--dest-compress-level** _format_ Specifies the compression level to use.  The value is specific to the compression algorithm used, e.g. for zstd the accepted values are in the range 1-20 (inclusive), while for gzip it is 1-9 (inclusive).  Algorithms that don't support the setting, will ignore it.
+
 ## EXAMPLES
 
 To copy the layers of the docker.io busybox image to a local directory:

go.mod

@@ -6,7 +6,7 @@ require (
 	github.com/VividCortex/ewma v1.1.1 // indirect
 	github.com/containerd/continuity v0.0.0-20180216233310-d8fb8589b0e8 // indirect
 	github.com/containers/buildah v1.8.4
-	github.com/containers/image v3.0.1+incompatible
+	github.com/containers/image v1.5.2-0.20190821113801-e003ccfc74de
 	github.com/containers/storage v1.13.0
 	github.com/docker/distribution v0.0.0-20170817175659-5f6282db7d65 // indirect
 	github.com/docker/docker v0.0.0-20180522102801-da99009bbb11

go.sum

@@ -17,6 +17,8 @@ github.com/containers/image v1.5.2-0.20190717062552-2178abd5f9b1 h1:RGlzwWSoGBbc
 github.com/containers/image v1.5.2-0.20190717062552-2178abd5f9b1/go.mod h1:8Vtij257IWSanUQKe1tAeNOm2sRVkSqQTVQ1IlwI3+M=
 github.com/containers/image v1.5.2-0.20190725091050-48acc3dcbb76 h1:+9unAKrV92Jvifb06UK8H4xTKf7h7XQDOsn4EC9eqH4=
 github.com/containers/image v1.5.2-0.20190725091050-48acc3dcbb76/go.mod h1:8Vtij257IWSanUQKe1tAeNOm2sRVkSqQTVQ1IlwI3+M=
+github.com/containers/image v1.5.2-0.20190821113801-e003ccfc74de h1:6M4DvYlNvVUJppB9rWaYFw30fgzt09z2dCj4VlOl3e4=
+github.com/containers/image v1.5.2-0.20190821113801-e003ccfc74de/go.mod h1:8Vtij257IWSanUQKe1tAeNOm2sRVkSqQTVQ1IlwI3+M=
 github.com/containers/image v2.0.0+incompatible h1:FTr6Br7jlIKNCKMjSOMbAxKp2keQ0//jzJaYNTVhauk=
 github.com/containers/image v2.0.0+incompatible/go.mod h1:8Vtij257IWSanUQKe1tAeNOm2sRVkSqQTVQ1IlwI3+M=
 github.com/containers/image v3.0.0+incompatible h1:pdUHY//H+3jYNnoTt+rqY8NsStX4ZBLKzPTlMC+XvnU=
@@ -49,6 +51,12 @@ github.com/etcd-io/bbolt v1.3.2 h1:RLRQ0TKLX7DlBRXAJHvbmXL17Q3KNnTBtZ9B6Qo+/Y0=
 github.com/etcd-io/bbolt v1.3.2/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw=
 github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680 h1:ZktWZesgun21uEDrwW7iEV1zPCGQldM2atlJZ3TdvVM=
 github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/giuseppe/image v0.0.0-20190723195852-da7a70cbb5f1 h1:j+wcaSLeY2xPBzeSU37obWjSdWvYjf8soPscPvmOGpY=
+github.com/giuseppe/image v0.0.0-20190723195852-da7a70cbb5f1/go.mod h1:t1Xf5SPi6pYsMJfQTMHfTpWgaQkRuy+0HMhAOJOj01E=
+github.com/giuseppe/image v0.0.0-20190806095912-8e15c8a868fc h1:qvH3jFNG7pyTd58rs/gNmtfTv7E9q8Xo7/RKLZn+zag=
+github.com/giuseppe/image v0.0.0-20190806095912-8e15c8a868fc/go.mod h1:t1Xf5SPi6pYsMJfQTMHfTpWgaQkRuy+0HMhAOJOj01E=
+github.com/giuseppe/image v0.0.0-20190813140229-9b055a514f21 h1:nUR9MenOi9gs8LlU5BSQ4zCmdTLc1Js5gT7rdymI3ZE=
+github.com/giuseppe/image v0.0.0-20190813140229-9b055a514f21/go.mod h1:t1Xf5SPi6pYsMJfQTMHfTpWgaQkRuy+0HMhAOJOj01E=
 github.com/go-check/check v0.0.0-20180628173108-788fd7840127 h1:0gkP6mzaMqkmpcJYCFOLkIBwI7xFExG03bbkOkCvUPI=
 github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98=
 github.com/gogo/protobuf v0.0.0-20170815085658-fcdc5011193f h1:r/AdTzqktq9nQpFlFePWcp+scVi+oFRajfjRJ3UnETg=

systemtest/020-copy.bats

@@ -44,6 +44,21 @@ function setup() {
     diff -urN $dir1 $dir2
 }
 
+# Compression zstd
+@test "copy: oci, round trip" {
+    local remote_image=docker://busybox:latest
+
+    local dir=$TESTDIR/dir
+
+    run_skopeo copy --dest-compress --dest-compress-format=zstd $remote_image oci:$dir:latest
+
+    # zstd magic number
+    local magic=$(printf "\x28\xb5\x2f\xfd")
+
+    # Check there is at least one file that has the zstd magic number as the first 4 bytes
+    (for i in $dir/blobs/sha256/*; do test "$(head -c 4 $i)" = $magic && exit 0; done; exit 1)
+}
+
 # Same image, extracted once with :tag and once without
 @test "copy: oci w/ and w/o tags" {
     local remote_image=docker://busybox:latest