From 926461ee70988c37cf777a5d02e47dc415d12f57 Mon Sep 17 00:00:00 2001 From: Valentin Rothberg Date: Fri, 26 Jul 2019 14:55:36 +0200 Subject: [PATCH] copy: set media types When copying an image, record the compression in the BlobInfo and use the information when updating the manifest's layer infos to set the layers' media types correctly. Note that consumers of the containers/image library need to update opencontainers/image-spec to commit 775207bd45b6cb8153ce218cc59351799217451f. Fixes: github.com/containers/libpod/issues/2013 Fixes: github.com/containers/buildah/issues/1589 Signed-off-by: Valentin Rothberg --- copy/copy.go | 6 + go.mod | 2 +- go.sum | 3 + image/docker_schema2.go | 18 +- image/docker_schema2_test.go | 32 ++ .../oci1-all-media-types-to-schema2.json | 41 ++ image/fixtures/oci1-all-media-types.json | 40 ++ image/fixtures/oci1-invalid-media-type.json | 15 + image/fixtures/oci1-to-schema2.json | 2 +- image/fixtures/schema2-all-media-types.json | 41 ++ .../fixtures/schema2-invalid-media-type.json | 36 ++ image/oci.go | 18 +- image/oci_test.go | 32 ++ image/sourced.go | 1 + manifest/docker_schema2.go | 56 ++- manifest/docker_schema2_test.go | 310 +++++++++++++++ .../ociv1.nondistributable.gzip.manifest.json | 19 + .../ociv1.nondistributable.manifest.json | 19 + .../ociv1.nondistributable.zstd.manifest.json | 19 + .../fixtures/ociv1.uncompressed.manifest.json | 29 ++ manifest/fixtures/ociv1.zstd.manifest.json | 29 ++ .../v2s2.nondistributable.gzip.manifest.json | 19 + .../v2s2.nondistributable.manifest.json | 19 + .../v2s2.nondistributable.zstd.manifest.json | 19 + .../fixtures/v2s2.uncompressed.manifest.json | 26 ++ manifest/fixtures/v2s2.zstd.manifest.json | 26 ++ manifest/manifest.go | 12 +- manifest/oci.go | 60 ++- manifest/oci_test.go | 358 ++++++++++++++++++ pkg/compression/compression.go | 73 ++-- storage/storage_image.go | 6 +- types/types.go | 36 +- 32 files changed, 1362 insertions(+), 60 deletions(-) create mode 100644 image/fixtures/oci1-all-media-types-to-schema2.json create mode 100644 image/fixtures/oci1-all-media-types.json create mode 100644 image/fixtures/oci1-invalid-media-type.json create mode 100644 image/fixtures/schema2-all-media-types.json create mode 100644 image/fixtures/schema2-invalid-media-type.json create mode 100644 manifest/docker_schema2_test.go create mode 100644 manifest/fixtures/ociv1.nondistributable.gzip.manifest.json create mode 100644 manifest/fixtures/ociv1.nondistributable.manifest.json create mode 100644 manifest/fixtures/ociv1.nondistributable.zstd.manifest.json create mode 100644 manifest/fixtures/ociv1.uncompressed.manifest.json create mode 100644 manifest/fixtures/ociv1.zstd.manifest.json create mode 100644 manifest/fixtures/v2s2.nondistributable.gzip.manifest.json create mode 100644 manifest/fixtures/v2s2.nondistributable.manifest.json create mode 100644 manifest/fixtures/v2s2.nondistributable.zstd.manifest.json create mode 100644 manifest/fixtures/v2s2.uncompressed.manifest.json create mode 100644 manifest/fixtures/v2s2.zstd.manifest.json create mode 100644 manifest/oci_test.go diff --git a/copy/copy.go b/copy/copy.go index 16c7900c67..6af46c6514 100644 --- a/copy/copy.go +++ b/copy/copy.go @@ -911,6 +911,12 @@ func (c *copier) copyBlobFromStream(ctx context.Context, srcStream io.Reader, sr return types.BlobInfo{}, errors.Wrap(err, "Error writing blob") } + uploadedInfo.CompressionOperation = compressionOperation + // If we can modify the layer's blob, set the desired algorithm for it to be set in the manifest. + if canModifyBlob && !isConfig { + uploadedInfo.CompressionAlgorithm = &desiredCompressionFormat + } + // This is fairly horrible: the writer from getOriginalLayerCopyWriter wants to consumer // all of the input (to compute DiffIDs), even if dest.PutBlob does not need it. // So, read everything from originalLayerReader, which will cause the rest to be diff --git a/go.mod b/go.mod index 8299df7455..452d5e4df9 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( github.com/mattn/go-isatty v0.0.4 // indirect github.com/mtrmac/gpgme v0.0.0-20170102180018-b2432428689c github.com/opencontainers/go-digest v1.0.0-rc1 - github.com/opencontainers/image-spec v1.0.0 + github.com/opencontainers/image-spec v1.0.2-0.20190823105129-775207bd45b6 github.com/opencontainers/selinux v1.2.2 github.com/ostreedev/ostree-go v0.0.0-20190702140239-759a8c1ac913 github.com/pkg/errors v0.8.1 diff --git a/go.sum b/go.sum index 2443bad61f..d8b738870f 100644 --- a/go.sum +++ b/go.sum @@ -75,6 +75,9 @@ github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2i github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/image-spec v1.0.0 h1:jcw3cCH887bLKETGYpv8afogdYchbShR0eH6oD9d5PQ= github.com/opencontainers/image-spec v1.0.0/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= +github.com/opencontainers/image-spec v1.0.2-0.20190823105129-775207bd45b6 h1:yN8BPXVwMBAm3Cuvh1L5XE8XpvYRMdsVLd82ILprhUU= +github.com/opencontainers/image-spec v1.0.2-0.20190823105129-775207bd45b6/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/runc v1.0.0-rc8 h1:dDCFes8Hj1r/i5qnypONo5jdOme/8HWZC/aNDyhECt0= github.com/opencontainers/runc v1.0.0-rc8/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/selinux v1.2.2 h1:Kx9J6eDG5/24A6DtUquGSpJQ+m2MUTahn4FtGEe8bFg= diff --git a/image/docker_schema2.go b/image/docker_schema2.go index 351e73ea1d..d3c663febd 100644 --- a/image/docker_schema2.go +++ b/image/docker_schema2.go @@ -6,6 +6,7 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "fmt" "io/ioutil" "strings" @@ -207,12 +208,21 @@ func (m *manifestSchema2) convertToManifestOCI1(ctx context.Context) (types.Imag layers := make([]imgspecv1.Descriptor, len(m.m.LayersDescriptors)) for idx := range layers { layers[idx] = oci1DescriptorFromSchema2Descriptor(m.m.LayersDescriptors[idx]) - if m.m.LayersDescriptors[idx].MediaType == manifest.DockerV2Schema2ForeignLayerMediaType { + switch m.m.LayersDescriptors[idx].MediaType { + case manifest.DockerV2Schema2ForeignLayerMediaType: layers[idx].MediaType = imgspecv1.MediaTypeImageLayerNonDistributable - } else { - // we assume layers are gzip'ed because docker v2s2 only deals with - // gzip'ed layers. However, OCI has non-gzip'ed layers as well. + case manifest.DockerV2Schema2ForeignLayerMediaTypeGzip: + layers[idx].MediaType = imgspecv1.MediaTypeImageLayerNonDistributableGzip + case manifest.DockerV2Schema2ForeignLayerMediaTypeZstd: + layers[idx].MediaType = imgspecv1.MediaTypeImageLayerNonDistributableZstd + case manifest.DockerV2SchemaLayerMediaTypeUncompressed: + layers[idx].MediaType = imgspecv1.MediaTypeImageLayer + case manifest.DockerV2Schema2LayerMediaType: layers[idx].MediaType = imgspecv1.MediaTypeImageLayerGzip + case manifest.DockerV2Schema2LayerMediaTypeZstd: + layers[idx].MediaType = imgspecv1.MediaTypeImageLayerZstd + default: + return nil, fmt.Errorf("Unknown media type during manifest conversion: %q", m.m.LayersDescriptors[idx].MediaType) } } diff --git a/image/docker_schema2_test.go b/image/docker_schema2_test.go index cb7179d368..751a5f06ff 100644 --- a/image/docker_schema2_test.go +++ b/image/docker_schema2_test.go @@ -520,6 +520,38 @@ func TestConvertToManifestOCI(t *testing.T) { assert.Equal(t, byHand, converted) } +func TestConvertToManifestOCIAllMediaTypes(t *testing.T) { + originalSrc := newSchema2ImageSource(t, "httpd-copy:latest") + original := manifestSchema2FromFixture(t, originalSrc, "schema2-all-media-types.json") + res, err := original.UpdatedImage(context.Background(), types.ManifestUpdateOptions{ + ManifestMIMEType: imgspecv1.MediaTypeImageManifest, + }) + require.NoError(t, err) + + convertedJSON, mt, err := res.Manifest(context.Background()) + require.NoError(t, err) + assert.Equal(t, imgspecv1.MediaTypeImageManifest, mt) + + byHandJSON, err := ioutil.ReadFile("fixtures/oci1-all-media-types.json") + require.NoError(t, err) + var converted, byHand map[string]interface{} + err = json.Unmarshal(byHandJSON, &byHand) + require.NoError(t, err) + err = json.Unmarshal(convertedJSON, &converted) + require.NoError(t, err) + assert.Equal(t, byHand, converted) +} + +func TestConvertToOCIWithInvalidMIMEType(t *testing.T) { + originalSrc := newSchema2ImageSource(t, "httpd-copy:latest") + original := manifestSchema2FromFixture(t, originalSrc, "schema2-invalid-media-type.json") + _, err := original.UpdatedImage(context.Background(), types.ManifestUpdateOptions{ + ManifestMIMEType: imgspecv1.MediaTypeImageManifest, + }) + require.Error(t, err) + require.Contains(t, err.Error(), "Unknown media type during manifest conversion: ") +} + func TestConvertToManifestSchema1(t *testing.T) { originalSrc := newSchema2ImageSource(t, "httpd-copy:latest") original := manifestSchema2FromFixture(t, originalSrc, "schema2.json") diff --git a/image/fixtures/oci1-all-media-types-to-schema2.json b/image/fixtures/oci1-all-media-types-to-schema2.json new file mode 100644 index 0000000000..702addfb0e --- /dev/null +++ b/image/fixtures/oci1-all-media-types-to-schema2.json @@ -0,0 +1,41 @@ +{ + "schemaVersion": 2, + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "config": { + "mediaType": "application/vnd.docker.container.image.v1+json", + "size": 4651, + "digest": "sha256:a13a0762ab7bed51a1b49adec0a702b1cd99294fd460a025b465bcfb7b152745" + }, + "layers": [ + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar", + "size": 51354364, + "digest": "sha256:6a5a5368e0c2d3e5909184fa28ddfd56072e7ff3ee9a945876f7eee5896ef5bb" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.zstd", + "size": 150, + "digest": "sha256:1bbf5d58d24c47512e234a5623474acf65ae00d4d1414272a893204f44cc680c" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 152, + "digest": "sha256:2bbf5d58d24c47512e234a5623474acf65ae00d4d1414272a893204f44cc680c" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.foreign.diff.tar", + "size": 11739507, + "digest": "sha256:8f5dc8a4b12c307ac84de90cdd9a7f3915d1be04c9388868ca118831099c67a9" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip", + "size": 8841833, + "digest": "sha256:bbd6b22eb11afce63cc76f6bc41042d99f10d6024c96b655dafba930b8d25909" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip", + "size": 291, + "digest": "sha256:960e52ecf8200cbd84e70eb2ad8678f4367e50d14357021872c10fa3fc5935fa" + } + ] +} \ No newline at end of file diff --git a/image/fixtures/oci1-all-media-types.json b/image/fixtures/oci1-all-media-types.json new file mode 100644 index 0000000000..1e57cfbe24 --- /dev/null +++ b/image/fixtures/oci1-all-media-types.json @@ -0,0 +1,40 @@ +{ + "schemaVersion": 2, + "config": { + "mediaType": "application/vnd.oci.image.config.v1+json", + "size": 4651, + "digest": "sha256:a13a0762ab7bed51a1b49adec0a702b1cd99294fd460a025b465bcfb7b152745" + }, + "layers": [ + { + "mediaType": "application/vnd.oci.image.layer.v1.tar", + "size": 51354364, + "digest": "sha256:6a5a5368e0c2d3e5909184fa28ddfd56072e7ff3ee9a945876f7eee5896ef5bb" + }, + { + "mediaType": "application/vnd.oci.image.layer.v1.tar+zstd", + "size": 150, + "digest": "sha256:1bbf5d58d24c47512e234a5623474acf65ae00d4d1414272a893204f44cc680c" + }, + { + "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", + "size": 152, + "digest": "sha256:2bbf5d58d24c47512e234a5623474acf65ae00d4d1414272a893204f44cc680c" + }, + { + "mediaType": "application/vnd.oci.image.layer.nondistributable.v1.tar", + "size": 11739507, + "digest": "sha256:8f5dc8a4b12c307ac84de90cdd9a7f3915d1be04c9388868ca118831099c67a9" + }, + { + "mediaType": "application/vnd.oci.image.layer.nondistributable.v1.tar+gzip", + "size": 8841833, + "digest": "sha256:bbd6b22eb11afce63cc76f6bc41042d99f10d6024c96b655dafba930b8d25909" + }, + { + "mediaType": "application/vnd.oci.image.layer.nondistributable.v1.tar+gzip", + "size": 291, + "digest": "sha256:960e52ecf8200cbd84e70eb2ad8678f4367e50d14357021872c10fa3fc5935fa" + } + ] +} \ No newline at end of file diff --git a/image/fixtures/oci1-invalid-media-type.json b/image/fixtures/oci1-invalid-media-type.json new file mode 100644 index 0000000000..7b7d06ee74 --- /dev/null +++ b/image/fixtures/oci1-invalid-media-type.json @@ -0,0 +1,15 @@ +{ + "schemaVersion": 2, + "config": { + "mediaType": "application/vnd.oci.image.config.v1+json", + "size": 5940, + "digest": "sha256:9ca4bda0a6b3727a6ffcc43e981cad0f24e2ec79d338f6ba325b4dfd0756fb8f" + }, + "layers": [ + { + "mediaType": "application/vnd.oci.image.layer.v1.tar+invalid-suffix", + "size": 51354364, + "digest": "sha256:6a5a5368e0c2d3e5909184fa28ddfd56072e7ff3ee9a945876f7eee5896ef5bb" + } + ] +} \ No newline at end of file diff --git a/image/fixtures/oci1-to-schema2.json b/image/fixtures/oci1-to-schema2.json index e0d72c09b9..50aa6dc06c 100644 --- a/image/fixtures/oci1-to-schema2.json +++ b/image/fixtures/oci1-to-schema2.json @@ -34,4 +34,4 @@ "digest": "sha256:960e52ecf8200cbd84e70eb2ad8678f4367e50d14357021872c10fa3fc5935fa" } ] -} +} \ No newline at end of file diff --git a/image/fixtures/schema2-all-media-types.json b/image/fixtures/schema2-all-media-types.json new file mode 100644 index 0000000000..2c987aa633 --- /dev/null +++ b/image/fixtures/schema2-all-media-types.json @@ -0,0 +1,41 @@ +{ + "schemaVersion": 2, + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "config": { + "mediaType": "application/vnd.docker.container.image.v1+json", + "size": 4651, + "digest": "sha256:9ca4bda0a6b3727a6ffcc43e981cad0f24e2ec79d338f6ba325b4dfd0756fb8f" + }, + "layers": [ + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar", + "size": 51354364, + "digest": "sha256:6a5a5368e0c2d3e5909184fa28ddfd56072e7ff3ee9a945876f7eee5896ef5bb" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.zstd", + "size": 150, + "digest": "sha256:1bbf5d58d24c47512e234a5623474acf65ae00d4d1414272a893204f44cc680c" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 152, + "digest": "sha256:2bbf5d58d24c47512e234a5623474acf65ae00d4d1414272a893204f44cc680c" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.foreign.diff.tar", + "size": 11739507, + "digest": "sha256:8f5dc8a4b12c307ac84de90cdd9a7f3915d1be04c9388868ca118831099c67a9" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip", + "size": 8841833, + "digest": "sha256:bbd6b22eb11afce63cc76f6bc41042d99f10d6024c96b655dafba930b8d25909" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip", + "size": 291, + "digest": "sha256:960e52ecf8200cbd84e70eb2ad8678f4367e50d14357021872c10fa3fc5935fa" + } + ] +} \ No newline at end of file diff --git a/image/fixtures/schema2-invalid-media-type.json b/image/fixtures/schema2-invalid-media-type.json new file mode 100644 index 0000000000..2edd532671 --- /dev/null +++ b/image/fixtures/schema2-invalid-media-type.json @@ -0,0 +1,36 @@ +{ + "schemaVersion": 2, + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "config": { + "mediaType": "application/octet-stream", + "size": 5940, + "digest": "sha256:9ca4bda0a6b3727a6ffcc43e981cad0f24e2ec79d338f6ba325b4dfd0756fb8f" + }, + "layers": [ + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.invalid", + "size": 51354364, + "digest": "sha256:6a5a5368e0c2d3e5909184fa28ddfd56072e7ff3ee9a945876f7eee5896ef5bb" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 150, + "digest": "sha256:1bbf5d58d24c47512e234a5623474acf65ae00d4d1414272a893204f44cc680c" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 11739507, + "digest": "sha256:8f5dc8a4b12c307ac84de90cdd9a7f3915d1be04c9388868ca118831099c67a9" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 8841833, + "digest": "sha256:bbd6b22eb11afce63cc76f6bc41042d99f10d6024c96b655dafba930b8d25909" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 291, + "digest": "sha256:960e52ecf8200cbd84e70eb2ad8678f4367e50d14357021872c10fa3fc5935fa" + } + ] + } \ No newline at end of file diff --git a/image/oci.go b/image/oci.go index cdff26e06a..e5cd468dd0 100644 --- a/image/oci.go +++ b/image/oci.go @@ -3,6 +3,7 @@ package image import ( "context" "encoding/json" + "fmt" "io/ioutil" "github.com/containers/image/docker/reference" @@ -187,7 +188,22 @@ func (m *manifestOCI1) convertToManifestSchema2() (types.Image, error) { layers := make([]manifest.Schema2Descriptor, len(m.m.Layers)) for idx := range layers { layers[idx] = schema2DescriptorFromOCI1Descriptor(m.m.Layers[idx]) - layers[idx].MediaType = manifest.DockerV2Schema2LayerMediaType + switch layers[idx].MediaType { + case imgspecv1.MediaTypeImageLayerNonDistributable: + layers[idx].MediaType = manifest.DockerV2Schema2ForeignLayerMediaType + case imgspecv1.MediaTypeImageLayerNonDistributableGzip: + layers[idx].MediaType = manifest.DockerV2Schema2ForeignLayerMediaTypeGzip + case imgspecv1.MediaTypeImageLayerNonDistributableZstd: + layers[idx].MediaType = manifest.DockerV2Schema2ForeignLayerMediaTypeZstd + case imgspecv1.MediaTypeImageLayer: + layers[idx].MediaType = manifest.DockerV2SchemaLayerMediaTypeUncompressed + case imgspecv1.MediaTypeImageLayerGzip: + layers[idx].MediaType = manifest.DockerV2Schema2LayerMediaType + case imgspecv1.MediaTypeImageLayerZstd: + layers[idx].MediaType = manifest.DockerV2Schema2LayerMediaTypeZstd + default: + return nil, fmt.Errorf("Unknown media type during manifest conversion: %q", layers[idx].MediaType) + } } // Rather than copying the ConfigBlob now, we just pass m.src to the diff --git a/image/oci_test.go b/image/oci_test.go index 69dc236c15..2850630097 100644 --- a/image/oci_test.go +++ b/image/oci_test.go @@ -410,3 +410,35 @@ func TestConvertToManifestSchema2(t *testing.T) { // FIXME? Test also the various failure cases, if only to see that we don't crash? } + +func TestConvertToManifestSchema2AllMediaTypes(t *testing.T) { + originalSrc := newOCI1ImageSource(t, "httpd-copy:latest") + original := manifestOCI1FromFixture(t, originalSrc, "oci1-all-media-types.json") + res, err := original.UpdatedImage(context.Background(), types.ManifestUpdateOptions{ + ManifestMIMEType: manifest.DockerV2Schema2MediaType, + }) + require.NoError(t, err) + + convertedJSON, mt, err := res.Manifest(context.Background()) + require.NoError(t, err) + assert.Equal(t, manifest.DockerV2Schema2MediaType, mt) + + byHandJSON, err := ioutil.ReadFile("fixtures/oci1-all-media-types-to-schema2.json") + require.NoError(t, err) + var converted, byHand map[string]interface{} + err = json.Unmarshal(byHandJSON, &byHand) + require.NoError(t, err) + err = json.Unmarshal(convertedJSON, &converted) + require.NoError(t, err) + assert.Equal(t, byHand, converted) +} + +func TestConvertToV2S2WithInvalidMIMEType(t *testing.T) { + originalSrc := newOCI1ImageSource(t, "httpd-copy:latest") + original := manifestOCI1FromFixture(t, originalSrc, "oci1-invalid-media-type.json") + _, err := original.UpdatedImage(context.Background(), types.ManifestUpdateOptions{ + ManifestMIMEType: manifest.DockerV2Schema2MediaType, + }) + require.Error(t, err) + require.Contains(t, err.Error(), "Unknown media type during manifest conversion: ") +} diff --git a/image/sourced.go b/image/sourced.go index 01cc28bbd2..c8364a1454 100644 --- a/image/sourced.go +++ b/image/sourced.go @@ -5,6 +5,7 @@ package image import ( "context" + "github.com/containers/image/types" ) diff --git a/manifest/docker_schema2.go b/manifest/docker_schema2.go index 76a80e5a6f..4f8a63f988 100644 --- a/manifest/docker_schema2.go +++ b/manifest/docker_schema2.go @@ -2,12 +2,15 @@ package manifest import ( "encoding/json" + "fmt" "time" + "github.com/containers/image/pkg/compression" "github.com/containers/image/pkg/strslice" "github.com/containers/image/types" "github.com/opencontainers/go-digest" "github.com/pkg/errors" + "github.com/sirupsen/logrus" ) // Schema2Descriptor is a “descriptor” in docker/distribution schema 2. @@ -207,7 +210,58 @@ func (m *Schema2) UpdateLayerInfos(layerInfos []types.BlobInfo) error { original := m.LayersDescriptors m.LayersDescriptors = make([]Schema2Descriptor, len(layerInfos)) for i, info := range layerInfos { - m.LayersDescriptors[i].MediaType = original[i].MediaType + // Set the correct media types based on the specified compression + // operation, the desired compression algorithm AND the original media + // type. + switch info.CompressionOperation { + case types.PreserveOriginal: + // Keep the original media type. + m.LayersDescriptors[i].MediaType = original[i].MediaType + + case types.Decompress: + // Decompress the original media type and check if it was + // non-distributable one or not. + switch original[i].MediaType { + case DockerV2Schema2ForeignLayerMediaTypeGzip, DockerV2Schema2ForeignLayerMediaTypeZstd: + m.LayersDescriptors[i].MediaType = DockerV2Schema2ForeignLayerMediaType + default: + m.LayersDescriptors[i].MediaType = DockerV2SchemaLayerMediaTypeUncompressed + } + + // TODO: should this only work on "known" media types? + // For background, please refer to: + // https://github.com/containers/image/pull/563#discussion_r316772562 + case types.Compress: + if info.CompressionAlgorithm == nil { + logrus.Debugf("Preparing updated manifest: blob %q was compressed but does not specify by which algorithm: falling back to use the original blob", info.Digest) + m.LayersDescriptors[i].MediaType = original[i].MediaType + break + } + // Compress the original media type and set the new one based on + // that type (distributable or not) and the specified compression + // algorithm. Throw an error if the algorithm is not supported. + switch info.CompressionAlgorithm.Name() { + case compression.Gzip.Name(): + switch original[i].MediaType { + case DockerV2Schema2ForeignLayerMediaType, DockerV2Schema2ForeignLayerMediaTypeZstd: + m.LayersDescriptors[i].MediaType = DockerV2Schema2ForeignLayerMediaTypeGzip + default: + m.LayersDescriptors[i].MediaType = DockerV2Schema2LayerMediaType + } + case compression.Zstd.Name(): + switch original[i].MediaType { + case DockerV2Schema2ForeignLayerMediaType, DockerV2Schema2ForeignLayerMediaTypeGzip: + m.LayersDescriptors[i].MediaType = DockerV2Schema2ForeignLayerMediaTypeZstd + default: + m.LayersDescriptors[i].MediaType = DockerV2Schema2LayerMediaTypeZstd + } + default: + return fmt.Errorf("Error preparing updated manifest: unknown compression algorithm %q fo layer %q", info.CompressionAlgorithm.Name(), info.Digest) + } + + default: + return fmt.Errorf("Error preparing updated manifest: unknown compression operation (%d) for layer %q", info.CompressionOperation, info.Digest) + } m.LayersDescriptors[i].Digest = info.Digest m.LayersDescriptors[i].Size = info.Size m.LayersDescriptors[i].URLs = info.URLs diff --git a/manifest/docker_schema2_test.go b/manifest/docker_schema2_test.go new file mode 100644 index 0000000000..1e1827f8e9 --- /dev/null +++ b/manifest/docker_schema2_test.go @@ -0,0 +1,310 @@ +package manifest + +import ( + "io/ioutil" + "testing" + + "github.com/containers/image/pkg/compression" + "github.com/containers/image/types" + "github.com/stretchr/testify/assert" +) + +func TestUpdateLayerInfosV2S2GzipToZstd(t *testing.T) { + bytes, err := ioutil.ReadFile("fixtures/v2s2.manifest.json") + assert.Nil(t, err) + + origManifest, err := Schema2FromManifest(bytes) + assert.Nil(t, err) + + err = origManifest.UpdateLayerInfos([]types.BlobInfo{ + { + Digest: "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f", + Size: 32654, + MediaType: DockerV2Schema2LayerMediaType, + CompressionOperation: types.Compress, + CompressionAlgorithm: &compression.Zstd, + }, + { + Digest: "sha256:3c3a4604a545cdc127456d94e421cd355bca5b528f4a9c1905b15da2eb4a4c6b", + Size: 16724, + MediaType: DockerV2Schema2LayerMediaType, + CompressionOperation: types.Compress, + CompressionAlgorithm: &compression.Zstd, + }, + { + Digest: "sha256:ec4b8955958665577945c89419d1af06b5f7636b4ac3da7f12184802ad867736", + Size: 73109, + MediaType: DockerV2Schema2LayerMediaType, + CompressionOperation: types.Compress, + CompressionAlgorithm: &compression.Zstd, + }, + }) + assert.Nil(t, err) + + updatedManifestBytes, err := origManifest.Serialize() + assert.Nil(t, err) + + bytes, err = ioutil.ReadFile("fixtures/v2s2.zstd.manifest.json") + assert.Nil(t, err) + + expectedManifest, err := Schema2FromManifest(bytes) + assert.Nil(t, err) + + expectedManifestBytes, err := expectedManifest.Serialize() + assert.Nil(t, err) + + assert.Equal(t, string(expectedManifestBytes), string(updatedManifestBytes)) +} + +func TestUpdateLayerInfosV2S2ZstdToGzip(t *testing.T) { + bytes, err := ioutil.ReadFile("fixtures/v2s2.zstd.manifest.json") + assert.Nil(t, err) + + origManifest, err := Schema2FromManifest(bytes) + assert.Nil(t, err) + + err = origManifest.UpdateLayerInfos([]types.BlobInfo{ + { + Digest: "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f", + Size: 32654, + MediaType: DockerV2Schema2LayerMediaTypeZstd, + CompressionOperation: types.Compress, + CompressionAlgorithm: &compression.Gzip, + }, + { + Digest: "sha256:3c3a4604a545cdc127456d94e421cd355bca5b528f4a9c1905b15da2eb4a4c6b", + Size: 16724, + MediaType: DockerV2Schema2LayerMediaTypeZstd, + CompressionOperation: types.Compress, + CompressionAlgorithm: &compression.Gzip, + }, + { + Digest: "sha256:ec4b8955958665577945c89419d1af06b5f7636b4ac3da7f12184802ad867736", + Size: 73109, + MediaType: DockerV2Schema2LayerMediaTypeZstd, + CompressionOperation: types.Compress, + CompressionAlgorithm: &compression.Gzip, + }, + }) + assert.Nil(t, err) + + updatedManifestBytes, err := origManifest.Serialize() + assert.Nil(t, err) + + bytes, err = ioutil.ReadFile("fixtures/v2s2.manifest.json") + assert.Nil(t, err) + + expectedManifest, err := Schema2FromManifest(bytes) + assert.Nil(t, err) + + expectedManifestBytes, err := expectedManifest.Serialize() + assert.Nil(t, err) + + assert.Equal(t, string(expectedManifestBytes), string(updatedManifestBytes)) +} + +func TestUpdateLayerInfosV2S2ZstdToUncompressed(t *testing.T) { + bytes, err := ioutil.ReadFile("fixtures/v2s2.zstd.manifest.json") + assert.Nil(t, err) + + origManifest, err := Schema2FromManifest(bytes) + assert.Nil(t, err) + + err = origManifest.UpdateLayerInfos([]types.BlobInfo{ + { + Digest: "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f", + Size: 32654, + MediaType: DockerV2Schema2LayerMediaTypeZstd, + CompressionOperation: types.Decompress, + }, + { + Digest: "sha256:3c3a4604a545cdc127456d94e421cd355bca5b528f4a9c1905b15da2eb4a4c6b", + Size: 16724, + MediaType: DockerV2Schema2LayerMediaTypeZstd, + CompressionOperation: types.Decompress, + }, + { + Digest: "sha256:ec4b8955958665577945c89419d1af06b5f7636b4ac3da7f12184802ad867736", + Size: 73109, + MediaType: DockerV2Schema2LayerMediaTypeZstd, + CompressionOperation: types.Decompress, + }, + }) + assert.Nil(t, err) + + updatedManifestBytes, err := origManifest.Serialize() + assert.Nil(t, err) + + bytes, err = ioutil.ReadFile("fixtures/v2s2.uncompressed.manifest.json") + assert.Nil(t, err) + + expectedManifest, err := Schema2FromManifest(bytes) + assert.Nil(t, err) + + expectedManifestBytes, err := expectedManifest.Serialize() + assert.Nil(t, err) + + assert.Equal(t, string(expectedManifestBytes), string(updatedManifestBytes)) +} + +func TestUpdateLayerInfosV2S2InvalidCompressionOperation(t *testing.T) { + bytes, err := ioutil.ReadFile("fixtures/v2s2.zstd.manifest.json") + assert.Nil(t, err) + + origManifest, err := Schema2FromManifest(bytes) + assert.Nil(t, err) + + err = origManifest.UpdateLayerInfos([]types.BlobInfo{ + { + Digest: "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f", + Size: 32654, + MediaType: DockerV2Schema2LayerMediaTypeZstd, + CompressionOperation: types.Decompress, + }, + { + Digest: "sha256:3c3a4604a545cdc127456d94e421cd355bca5b528f4a9c1905b15da2eb4a4c6b", + Size: 16724, + MediaType: DockerV2Schema2LayerMediaTypeZstd, + CompressionOperation: types.Decompress, + }, + { + Digest: "sha256:ec4b8955958665577945c89419d1af06b5f7636b4ac3da7f12184802ad867736", + Size: 73109, + MediaType: DockerV2Schema2LayerMediaTypeZstd, + CompressionOperation: 42, // MUST fail here + }, + }) + assert.NotNil(t, err) +} + +func TestUpdateLayerInfosV2S2InvalidCompressionAlgorithm(t *testing.T) { + bytes, err := ioutil.ReadFile("fixtures/v2s2.zstd.manifest.json") + assert.Nil(t, err) + + origManifest, err := Schema2FromManifest(bytes) + assert.Nil(t, err) + + customCompression := compression.Algorithm{} + err = origManifest.UpdateLayerInfos([]types.BlobInfo{ + { + Digest: "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f", + Size: 32654, + MediaType: DockerV2Schema2LayerMediaTypeZstd, + CompressionOperation: types.Compress, + CompressionAlgorithm: &compression.Zstd, + }, + { + Digest: "sha256:3c3a4604a545cdc127456d94e421cd355bca5b528f4a9c1905b15da2eb4a4c6b", + Size: 16724, + MediaType: DockerV2Schema2LayerMediaTypeZstd, + CompressionOperation: types.Compress, + CompressionAlgorithm: &compression.Zstd, + }, + { + Digest: "sha256:ec4b8955958665577945c89419d1af06b5f7636b4ac3da7f12184802ad867736", + Size: 73109, + MediaType: DockerV2Schema2LayerMediaTypeZstd, + CompressionOperation: types.Compress, + CompressionAlgorithm: &customCompression, // MUST fail here + }, + }) + assert.NotNil(t, err) +} + +func TestUpdateLayerInfosV2S2NondistributableToGzip(t *testing.T) { + bytes, err := ioutil.ReadFile("fixtures/v2s2.nondistributable.manifest.json") + assert.Nil(t, err) + + origManifest, err := Schema2FromManifest(bytes) + assert.Nil(t, err) + + err = origManifest.UpdateLayerInfos([]types.BlobInfo{ + { + Digest: "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f", + Size: 32654, + MediaType: DockerV2Schema2ForeignLayerMediaType, + CompressionOperation: types.Compress, + CompressionAlgorithm: &compression.Gzip, + }, + }) + assert.Nil(t, err) + + updatedManifestBytes, err := origManifest.Serialize() + assert.Nil(t, err) + + bytes, err = ioutil.ReadFile("fixtures/v2s2.nondistributable.gzip.manifest.json") + assert.Nil(t, err) + + expectedManifest, err := Schema2FromManifest(bytes) + assert.Nil(t, err) + + expectedManifestBytes, err := expectedManifest.Serialize() + assert.Nil(t, err) + + assert.Equal(t, string(expectedManifestBytes), string(updatedManifestBytes)) +} + +func TestUpdateLayerInfosV2S2NondistributableToZstd(t *testing.T) { + bytes, err := ioutil.ReadFile("fixtures/v2s2.nondistributable.manifest.json") + assert.Nil(t, err) + + origManifest, err := Schema2FromManifest(bytes) + assert.Nil(t, err) + + err = origManifest.UpdateLayerInfos([]types.BlobInfo{ + { + Digest: "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f", + Size: 32654, + MediaType: DockerV2Schema2ForeignLayerMediaType, + CompressionOperation: types.Compress, + CompressionAlgorithm: &compression.Zstd, + }, + }) + assert.Nil(t, err) + + updatedManifestBytes, err := origManifest.Serialize() + assert.Nil(t, err) + + bytes, err = ioutil.ReadFile("fixtures/v2s2.nondistributable.zstd.manifest.json") + assert.Nil(t, err) + + expectedManifest, err := Schema2FromManifest(bytes) + assert.Nil(t, err) + + expectedManifestBytes, err := expectedManifest.Serialize() + assert.Nil(t, err) + + assert.Equal(t, string(expectedManifestBytes), string(updatedManifestBytes)) +} + +func TestUpdateLayerInfosV2S2NondistributableGzipToUncompressed(t *testing.T) { + bytes, err := ioutil.ReadFile("fixtures/v2s2.nondistributable.gzip.manifest.json") + assert.Nil(t, err) + + origManifest, err := Schema2FromManifest(bytes) + assert.Nil(t, err) + + err = origManifest.UpdateLayerInfos([]types.BlobInfo{ + { + Digest: "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f", + Size: 32654, + MediaType: DockerV2Schema2ForeignLayerMediaType, + CompressionOperation: types.Decompress, + }, + }) + assert.Nil(t, err) + + updatedManifestBytes, err := origManifest.Serialize() + assert.Nil(t, err) + + bytes, err = ioutil.ReadFile("fixtures/v2s2.nondistributable.manifest.json") + assert.Nil(t, err) + + expectedManifest, err := Schema2FromManifest(bytes) + assert.Nil(t, err) + + expectedManifestBytes, err := expectedManifest.Serialize() + assert.Nil(t, err) + + assert.Equal(t, string(expectedManifestBytes), string(updatedManifestBytes)) +} diff --git a/manifest/fixtures/ociv1.nondistributable.gzip.manifest.json b/manifest/fixtures/ociv1.nondistributable.gzip.manifest.json new file mode 100644 index 0000000000..2729e03a0a --- /dev/null +++ b/manifest/fixtures/ociv1.nondistributable.gzip.manifest.json @@ -0,0 +1,19 @@ +{ + "schemaVersion": 2, + "config": { + "mediaType": "application/vnd.oci.image.config.v1+json", + "size": 7023, + "digest": "sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7" + }, + "layers": [ + { + "mediaType": "application/vnd.oci.image.layer.nondistributable.v1.tar+gzip", + "size": 32654, + "digest": "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f" + } + ], + "annotations": { + "com.example.key1": "value1", + "com.example.key2": "value2" + } +} \ No newline at end of file diff --git a/manifest/fixtures/ociv1.nondistributable.manifest.json b/manifest/fixtures/ociv1.nondistributable.manifest.json new file mode 100644 index 0000000000..73b20b9b54 --- /dev/null +++ b/manifest/fixtures/ociv1.nondistributable.manifest.json @@ -0,0 +1,19 @@ +{ + "schemaVersion": 2, + "config": { + "mediaType": "application/vnd.oci.image.config.v1+json", + "size": 7023, + "digest": "sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7" + }, + "layers": [ + { + "mediaType": "application/vnd.oci.image.layer.nondistributable.v1.tar", + "size": 32654, + "digest": "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f" + } + ], + "annotations": { + "com.example.key1": "value1", + "com.example.key2": "value2" + } +} \ No newline at end of file diff --git a/manifest/fixtures/ociv1.nondistributable.zstd.manifest.json b/manifest/fixtures/ociv1.nondistributable.zstd.manifest.json new file mode 100644 index 0000000000..948f6f0c5e --- /dev/null +++ b/manifest/fixtures/ociv1.nondistributable.zstd.manifest.json @@ -0,0 +1,19 @@ +{ + "schemaVersion": 2, + "config": { + "mediaType": "application/vnd.oci.image.config.v1+json", + "size": 7023, + "digest": "sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7" + }, + "layers": [ + { + "mediaType": "application/vnd.oci.image.layer.nondistributable.v1.tar+zstd", + "size": 32654, + "digest": "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f" + } + ], + "annotations": { + "com.example.key1": "value1", + "com.example.key2": "value2" + } +} \ No newline at end of file diff --git a/manifest/fixtures/ociv1.uncompressed.manifest.json b/manifest/fixtures/ociv1.uncompressed.manifest.json new file mode 100644 index 0000000000..206165c965 --- /dev/null +++ b/manifest/fixtures/ociv1.uncompressed.manifest.json @@ -0,0 +1,29 @@ +{ + "schemaVersion": 2, + "config": { + "mediaType": "application/vnd.oci.image.config.v1+json", + "size": 7023, + "digest": "sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7" + }, + "layers": [ + { + "mediaType": "application/vnd.oci.image.layer.v1.tar", + "size": 32654, + "digest": "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f" + }, + { + "mediaType": "application/vnd.oci.image.layer.v1.tar", + "size": 16724, + "digest": "sha256:3c3a4604a545cdc127456d94e421cd355bca5b528f4a9c1905b15da2eb4a4c6b" + }, + { + "mediaType": "application/vnd.oci.image.layer.v1.tar", + "size": 73109, + "digest": "sha256:ec4b8955958665577945c89419d1af06b5f7636b4ac3da7f12184802ad867736" + } + ], + "annotations": { + "com.example.key1": "value1", + "com.example.key2": "value2" + } +} \ No newline at end of file diff --git a/manifest/fixtures/ociv1.zstd.manifest.json b/manifest/fixtures/ociv1.zstd.manifest.json new file mode 100644 index 0000000000..c2a3ca13c1 --- /dev/null +++ b/manifest/fixtures/ociv1.zstd.manifest.json @@ -0,0 +1,29 @@ +{ + "schemaVersion": 2, + "config": { + "mediaType": "application/vnd.oci.image.config.v1+json", + "size": 7023, + "digest": "sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7" + }, + "layers": [ + { + "mediaType": "application/vnd.oci.image.layer.v1.tar+zstd", + "size": 32654, + "digest": "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f" + }, + { + "mediaType": "application/vnd.oci.image.layer.v1.tar+zstd", + "size": 16724, + "digest": "sha256:3c3a4604a545cdc127456d94e421cd355bca5b528f4a9c1905b15da2eb4a4c6b" + }, + { + "mediaType": "application/vnd.oci.image.layer.v1.tar+zstd", + "size": 73109, + "digest": "sha256:ec4b8955958665577945c89419d1af06b5f7636b4ac3da7f12184802ad867736" + } + ], + "annotations": { + "com.example.key1": "value1", + "com.example.key2": "value2" + } +} \ No newline at end of file diff --git a/manifest/fixtures/v2s2.nondistributable.gzip.manifest.json b/manifest/fixtures/v2s2.nondistributable.gzip.manifest.json new file mode 100644 index 0000000000..f50552bd31 --- /dev/null +++ b/manifest/fixtures/v2s2.nondistributable.gzip.manifest.json @@ -0,0 +1,19 @@ +{ + "schemaVersion": 2, + "config": { + "mediaType": "application/vnd.oci.image.config.v1+json", + "size": 7023, + "digest": "sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7" + }, + "layers": [ + { + "mediaType": "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip", + "size": 32654, + "digest": "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f" + } + ], + "annotations": { + "com.example.key1": "value1", + "com.example.key2": "value2" + } +} \ No newline at end of file diff --git a/manifest/fixtures/v2s2.nondistributable.manifest.json b/manifest/fixtures/v2s2.nondistributable.manifest.json new file mode 100644 index 0000000000..d0edb267f0 --- /dev/null +++ b/manifest/fixtures/v2s2.nondistributable.manifest.json @@ -0,0 +1,19 @@ +{ + "schemaVersion": 2, + "config": { + "mediaType": "application/vnd.oci.image.config.v1+json", + "size": 7023, + "digest": "sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7" + }, + "layers": [ + { + "mediaType": "application/vnd.docker.image.rootfs.foreign.diff.tar", + "size": 32654, + "digest": "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f" + } + ], + "annotations": { + "com.example.key1": "value1", + "com.example.key2": "value2" + } +} \ No newline at end of file diff --git a/manifest/fixtures/v2s2.nondistributable.zstd.manifest.json b/manifest/fixtures/v2s2.nondistributable.zstd.manifest.json new file mode 100644 index 0000000000..1ae0087bf0 --- /dev/null +++ b/manifest/fixtures/v2s2.nondistributable.zstd.manifest.json @@ -0,0 +1,19 @@ +{ + "schemaVersion": 2, + "config": { + "mediaType": "application/vnd.oci.image.config.v1+json", + "size": 7023, + "digest": "sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7" + }, + "layers": [ + { + "mediaType": "application/vnd.docker.image.rootfs.foreign.diff.tar.zstd", + "size": 32654, + "digest": "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f" + } + ], + "annotations": { + "com.example.key1": "value1", + "com.example.key2": "value2" + } +} \ No newline at end of file diff --git a/manifest/fixtures/v2s2.uncompressed.manifest.json b/manifest/fixtures/v2s2.uncompressed.manifest.json new file mode 100644 index 0000000000..869d97e94a --- /dev/null +++ b/manifest/fixtures/v2s2.uncompressed.manifest.json @@ -0,0 +1,26 @@ +{ + "schemaVersion": 2, + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "config": { + "mediaType": "application/vnd.docker.container.image.v1+json", + "size": 7023, + "digest": "sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7" + }, + "layers": [ + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar", + "size": 32654, + "digest": "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar", + "size": 16724, + "digest": "sha256:3c3a4604a545cdc127456d94e421cd355bca5b528f4a9c1905b15da2eb4a4c6b" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar", + "size": 73109, + "digest": "sha256:ec4b8955958665577945c89419d1af06b5f7636b4ac3da7f12184802ad867736" + } + ] +} \ No newline at end of file diff --git a/manifest/fixtures/v2s2.zstd.manifest.json b/manifest/fixtures/v2s2.zstd.manifest.json new file mode 100644 index 0000000000..332ae2a8cb --- /dev/null +++ b/manifest/fixtures/v2s2.zstd.manifest.json @@ -0,0 +1,26 @@ +{ + "schemaVersion": 2, + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "config": { + "mediaType": "application/vnd.docker.container.image.v1+json", + "size": 7023, + "digest": "sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7" + }, + "layers": [ + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.zstd", + "size": 32654, + "digest": "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.zstd", + "size": 16724, + "digest": "sha256:3c3a4604a545cdc127456d94e421cd355bca5b528f4a9c1905b15da2eb4a4c6b" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.zstd", + "size": 73109, + "digest": "sha256:ec4b8955958665577945c89419d1af06b5f7636b4ac3da7f12184802ad867736" + } + ] +} \ No newline at end of file diff --git a/manifest/manifest.go b/manifest/manifest.go index ae1921b6cc..d2113455c0 100644 --- a/manifest/manifest.go +++ b/manifest/manifest.go @@ -12,7 +12,7 @@ import ( // FIXME: Should we just use docker/distribution and docker/docker implementations directly? -// FIXME(runcom, mitr): should we havea mediatype pkg?? +// FIXME(runcom, mitr): should we have a mediatype pkg?? const ( // DockerV2Schema1MediaType MIME type represents Docker manifest schema 1 DockerV2Schema1MediaType = "application/vnd.docker.distribution.manifest.v1+json" @@ -24,10 +24,18 @@ const ( DockerV2Schema2ConfigMediaType = "application/vnd.docker.container.image.v1+json" // DockerV2Schema2LayerMediaType is the MIME type used for schema 2 layers. DockerV2Schema2LayerMediaType = "application/vnd.docker.image.rootfs.diff.tar.gzip" + // DockerV2Schema2LayerMediaTypeZstd is the MIME type used for schema 2 layers compressed with zstd. + DockerV2Schema2LayerMediaTypeZstd = "application/vnd.docker.image.rootfs.diff.tar.zstd" + // DockerV2SchemaLayerMediaTypeUncompressed is the mediaType used for uncompressed layers. + DockerV2SchemaLayerMediaTypeUncompressed = "application/vnd.docker.image.rootfs.diff.tar" // DockerV2ListMediaType MIME type represents Docker manifest schema 2 list DockerV2ListMediaType = "application/vnd.docker.distribution.manifest.list.v2+json" // DockerV2Schema2ForeignLayerMediaType is the MIME type used for schema 2 foreign layers. - DockerV2Schema2ForeignLayerMediaType = "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip" + DockerV2Schema2ForeignLayerMediaType = "application/vnd.docker.image.rootfs.foreign.diff.tar" + // DockerV2Schema2ForeignLayerMediaType is the MIME type used for gzippped schema 2 foreign layers. + DockerV2Schema2ForeignLayerMediaTypeGzip = "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip" + // DockerV2Schema2ForeignLayerMediaType is the MIME type used for schema 2 foreign layers compressed with zstd. + DockerV2Schema2ForeignLayerMediaTypeZstd = "application/vnd.docker.image.rootfs.foreign.diff.tar.zstd" ) // DefaultRequestedManifestMIMETypes is a list of MIME types a types.ImageSource diff --git a/manifest/oci.go b/manifest/oci.go index dd65e0ba27..a3818c0efc 100644 --- a/manifest/oci.go +++ b/manifest/oci.go @@ -2,12 +2,15 @@ package manifest import ( "encoding/json" + "fmt" + "github.com/containers/image/pkg/compression" "github.com/containers/image/types" "github.com/opencontainers/go-digest" "github.com/opencontainers/image-spec/specs-go" imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" + "github.com/sirupsen/logrus" ) // BlobInfoFromOCI1Descriptor returns a types.BlobInfo based on the input OCI1 descriptor. @@ -81,7 +84,62 @@ func (m *OCI1) UpdateLayerInfos(layerInfos []types.BlobInfo) error { original := m.Layers m.Layers = make([]imgspecv1.Descriptor, len(layerInfos)) for i, info := range layerInfos { - m.Layers[i].MediaType = original[i].MediaType + // Set the correct media types based on the specified compression + // operation, the desired compression algorithm AND the original media + // type. + switch info.CompressionOperation { + case types.PreserveOriginal: + // Keep the original media type. + m.Layers[i].MediaType = original[i].MediaType + + case types.Decompress: + // Decompress the original media type and check if it was + // non-distributable one or not. + switch original[i].MediaType { + case imgspecv1.MediaTypeImageLayerNonDistributableGzip, imgspecv1.MediaTypeImageLayerNonDistributableZstd: + m.Layers[i].MediaType = imgspecv1.MediaTypeImageLayerNonDistributable + default: + m.Layers[i].MediaType = imgspecv1.MediaTypeImageLayer + } + + // TODO: should this only work on "known" media types? + // For background, please refer to: + // https://github.com/containers/image/pull/563#discussion_r316772562 + case types.Compress: + if info.CompressionAlgorithm == nil { + logrus.Debugf("Preparing updated manifest: blob %q was compressed but does not specify by which algorithm: falling back to use the original blob", info.Digest) + m.Layers[i].MediaType = original[i].MediaType + break + } + // Compress the original media type and set the new one based on + // that type (distributable or not) and the specified compression + // algorithm. Throw an error if the algorithm is not supported. + switch info.CompressionAlgorithm.Name() { + case compression.Gzip.Name(): + switch original[i].MediaType { + case imgspecv1.MediaTypeImageLayerNonDistributable, imgspecv1.MediaTypeImageLayerNonDistributableZstd: + m.Layers[i].MediaType = imgspecv1.MediaTypeImageLayerNonDistributableGzip + + default: + m.Layers[i].MediaType = imgspecv1.MediaTypeImageLayerGzip + } + + case compression.Zstd.Name(): + switch original[i].MediaType { + case imgspecv1.MediaTypeImageLayerNonDistributable, imgspecv1.MediaTypeImageLayerNonDistributableGzip: + m.Layers[i].MediaType = imgspecv1.MediaTypeImageLayerNonDistributableZstd + + default: + m.Layers[i].MediaType = imgspecv1.MediaTypeImageLayerZstd + } + + default: + return fmt.Errorf("Error preparing updated manifest: unknown compression algorithm %q for layer %q", info.CompressionAlgorithm.Name(), info.Digest) + } + + default: + return fmt.Errorf("Error preparing updated manifest: unknown compression operation (%d) for layer %q", info.CompressionOperation, info.Digest) + } m.Layers[i].Digest = info.Digest m.Layers[i].Size = info.Size m.Layers[i].Annotations = info.Annotations diff --git a/manifest/oci_test.go b/manifest/oci_test.go new file mode 100644 index 0000000000..a6f17fbbb2 --- /dev/null +++ b/manifest/oci_test.go @@ -0,0 +1,358 @@ +package manifest + +import ( + "io/ioutil" + "testing" + + "github.com/containers/image/pkg/compression" + "github.com/containers/image/types" + imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/stretchr/testify/assert" +) + +func TestUpdateLayerInfosOCIGzipToZstd(t *testing.T) { + bytes, err := ioutil.ReadFile("fixtures/ociv1.manifest.json") + assert.Nil(t, err) + + manifest, err := OCI1FromManifest(bytes) + assert.Nil(t, err) + + err = manifest.UpdateLayerInfos([]types.BlobInfo{ + { + Digest: "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f", + Size: 32654, + MediaType: imgspecv1.MediaTypeImageLayerGzip, + CompressionOperation: types.Compress, + CompressionAlgorithm: &compression.Zstd, + }, + { + Digest: "sha256:3c3a4604a545cdc127456d94e421cd355bca5b528f4a9c1905b15da2eb4a4c6b", + Size: 16724, + MediaType: imgspecv1.MediaTypeImageLayerGzip, + CompressionOperation: types.Compress, + CompressionAlgorithm: &compression.Zstd, + }, + { + Digest: "sha256:ec4b8955958665577945c89419d1af06b5f7636b4ac3da7f12184802ad867736", + Size: 73109, + MediaType: imgspecv1.MediaTypeImageLayerGzip, + CompressionOperation: types.Compress, + CompressionAlgorithm: &compression.Zstd, + }, + }) + assert.Nil(t, err) + + updatedManifestBytes, err := manifest.Serialize() + assert.Nil(t, err) + + bytes, err = ioutil.ReadFile("fixtures/ociv1.zstd.manifest.json") + assert.Nil(t, err) + + expectedManifest, err := OCI1FromManifest(bytes) + assert.Nil(t, err) + + expectedManifestBytes, err := expectedManifest.Serialize() + assert.Nil(t, err) + + assert.Equal(t, string(expectedManifestBytes), string(updatedManifestBytes)) +} + +func TestUpdateLayerInfosOCIZstdToGzip(t *testing.T) { + bytes, err := ioutil.ReadFile("fixtures/ociv1.zstd.manifest.json") + assert.Nil(t, err) + + manifest, err := OCI1FromManifest(bytes) + assert.Nil(t, err) + + err = manifest.UpdateLayerInfos([]types.BlobInfo{ + { + Digest: "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f", + Size: 32654, + MediaType: imgspecv1.MediaTypeImageLayerZstd, + CompressionOperation: types.Compress, + CompressionAlgorithm: &compression.Gzip, + }, + { + Digest: "sha256:3c3a4604a545cdc127456d94e421cd355bca5b528f4a9c1905b15da2eb4a4c6b", + Size: 16724, + MediaType: imgspecv1.MediaTypeImageLayerZstd, + CompressionOperation: types.Compress, + CompressionAlgorithm: &compression.Gzip, + }, + { + Digest: "sha256:ec4b8955958665577945c89419d1af06b5f7636b4ac3da7f12184802ad867736", + Size: 73109, + MediaType: imgspecv1.MediaTypeImageLayerZstd, + CompressionOperation: types.Compress, + CompressionAlgorithm: &compression.Gzip, + }, + }) + assert.Nil(t, err) + + updatedManifestBytes, err := manifest.Serialize() + assert.Nil(t, err) + + bytes, err = ioutil.ReadFile("fixtures/ociv1.manifest.json") + assert.Nil(t, err) + + expectedManifest, err := OCI1FromManifest(bytes) + assert.Nil(t, err) + + expectedManifestBytes, err := expectedManifest.Serialize() + assert.Nil(t, err) + + assert.Equal(t, string(expectedManifestBytes), string(updatedManifestBytes)) +} + +func TestUpdateLayerInfosOCIZstdToUncompressed(t *testing.T) { + bytes, err := ioutil.ReadFile("fixtures/ociv1.zstd.manifest.json") + assert.Nil(t, err) + + manifest, err := OCI1FromManifest(bytes) + assert.Nil(t, err) + + err = manifest.UpdateLayerInfos([]types.BlobInfo{ + { + Digest: "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f", + Size: 32654, + MediaType: imgspecv1.MediaTypeImageLayerZstd, + CompressionOperation: types.Decompress, + }, + { + Digest: "sha256:3c3a4604a545cdc127456d94e421cd355bca5b528f4a9c1905b15da2eb4a4c6b", + Size: 16724, + MediaType: imgspecv1.MediaTypeImageLayerZstd, + CompressionOperation: types.Decompress, + }, + { + Digest: "sha256:ec4b8955958665577945c89419d1af06b5f7636b4ac3da7f12184802ad867736", + Size: 73109, + MediaType: imgspecv1.MediaTypeImageLayerZstd, + CompressionOperation: types.Decompress, + }, + }) + assert.Nil(t, err) + + updatedManifestBytes, err := manifest.Serialize() + assert.Nil(t, err) + + bytes, err = ioutil.ReadFile("fixtures/ociv1.uncompressed.manifest.json") + assert.Nil(t, err) + + expectedManifest, err := OCI1FromManifest(bytes) + assert.Nil(t, err) + + expectedManifestBytes, err := expectedManifest.Serialize() + assert.Nil(t, err) + + assert.Equal(t, string(expectedManifestBytes), string(updatedManifestBytes)) +} + +func TestUpdateLayerInfosInvalidCompressionOperation(t *testing.T) { + bytes, err := ioutil.ReadFile("fixtures/ociv1.zstd.manifest.json") + assert.Nil(t, err) + + manifest, err := OCI1FromManifest(bytes) + assert.Nil(t, err) + + err = manifest.UpdateLayerInfos([]types.BlobInfo{ + { + Digest: "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f", + Size: 32654, + MediaType: imgspecv1.MediaTypeImageLayerZstd, + CompressionOperation: types.Compress, + CompressionAlgorithm: &compression.Gzip, + }, + { + Digest: "sha256:3c3a4604a545cdc127456d94e421cd355bca5b528f4a9c1905b15da2eb4a4c6b", + Size: 16724, + MediaType: imgspecv1.MediaTypeImageLayerZstd, + CompressionOperation: 42, // MUST fail here + CompressionAlgorithm: &compression.Gzip, + }, + { + Digest: "sha256:ec4b8955958665577945c89419d1af06b5f7636b4ac3da7f12184802ad867736", + Size: 73109, + MediaType: imgspecv1.MediaTypeImageLayerZstd, + CompressionOperation: types.Compress, + CompressionAlgorithm: &compression.Gzip, + }, + }) + assert.NotNil(t, err) +} + +func TestUpdateLayerInfosInvalidCompressionAlgorithm(t *testing.T) { + bytes, err := ioutil.ReadFile("fixtures/ociv1.zstd.manifest.json") + assert.Nil(t, err) + + manifest, err := OCI1FromManifest(bytes) + assert.Nil(t, err) + + customCompression := compression.Algorithm{} + err = manifest.UpdateLayerInfos([]types.BlobInfo{ + { + Digest: "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f", + Size: 32654, + MediaType: imgspecv1.MediaTypeImageLayerZstd, + CompressionOperation: types.Compress, + CompressionAlgorithm: &compression.Gzip, + }, + { + Digest: "sha256:3c3a4604a545cdc127456d94e421cd355bca5b528f4a9c1905b15da2eb4a4c6b", + Size: 16724, + MediaType: imgspecv1.MediaTypeImageLayerZstd, + CompressionOperation: 42, + CompressionAlgorithm: &compression.Gzip, + }, + { + Digest: "sha256:ec4b8955958665577945c89419d1af06b5f7636b4ac3da7f12184802ad867736", + Size: 73109, + MediaType: imgspecv1.MediaTypeImageLayerZstd, + CompressionOperation: types.Compress, + CompressionAlgorithm: &customCompression, // MUST fail here + }, + }) + assert.NotNil(t, err) +} + +func TestUpdateLayerInfosOCIGzipToUncompressed(t *testing.T) { + bytes, err := ioutil.ReadFile("fixtures/ociv1.manifest.json") + assert.Nil(t, err) + + manifest, err := OCI1FromManifest(bytes) + assert.Nil(t, err) + + err = manifest.UpdateLayerInfos([]types.BlobInfo{ + { + Digest: "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f", + Size: 32654, + MediaType: imgspecv1.MediaTypeImageLayerGzip, + CompressionOperation: types.Decompress, + }, + { + Digest: "sha256:3c3a4604a545cdc127456d94e421cd355bca5b528f4a9c1905b15da2eb4a4c6b", + Size: 16724, + MediaType: imgspecv1.MediaTypeImageLayerGzip, + CompressionOperation: types.Decompress, + }, + { + Digest: "sha256:ec4b8955958665577945c89419d1af06b5f7636b4ac3da7f12184802ad867736", + Size: 73109, + MediaType: imgspecv1.MediaTypeImageLayerGzip, + CompressionOperation: types.Decompress, + }, + }) + assert.Nil(t, err) + + updatedManifestBytes, err := manifest.Serialize() + assert.Nil(t, err) + + bytes, err = ioutil.ReadFile("fixtures/ociv1.uncompressed.manifest.json") + assert.Nil(t, err) + + expectedManifest, err := OCI1FromManifest(bytes) + assert.Nil(t, err) + + expectedManifestBytes, err := expectedManifest.Serialize() + assert.Nil(t, err) + + assert.Equal(t, string(expectedManifestBytes), string(updatedManifestBytes)) +} + +func TestUpdateLayerInfosOCINondistributableToGzip(t *testing.T) { + bytes, err := ioutil.ReadFile("fixtures/ociv1.nondistributable.manifest.json") + assert.Nil(t, err) + + manifest, err := OCI1FromManifest(bytes) + assert.Nil(t, err) + + err = manifest.UpdateLayerInfos([]types.BlobInfo{ + { + Digest: "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f", + Size: 32654, + MediaType: imgspecv1.MediaTypeImageLayerGzip, + CompressionOperation: types.Compress, + CompressionAlgorithm: &compression.Gzip, + }, + }) + assert.Nil(t, err) + + updatedManifestBytes, err := manifest.Serialize() + assert.Nil(t, err) + + bytes, err = ioutil.ReadFile("fixtures/ociv1.nondistributable.gzip.manifest.json") + assert.Nil(t, err) + + expectedManifest, err := OCI1FromManifest(bytes) + assert.Nil(t, err) + + expectedManifestBytes, err := expectedManifest.Serialize() + assert.Nil(t, err) + + assert.Equal(t, string(expectedManifestBytes), string(updatedManifestBytes)) +} + +func TestUpdateLayerInfosOCINondistributableToZstd(t *testing.T) { + bytes, err := ioutil.ReadFile("fixtures/ociv1.nondistributable.manifest.json") + assert.Nil(t, err) + + manifest, err := OCI1FromManifest(bytes) + assert.Nil(t, err) + + err = manifest.UpdateLayerInfos([]types.BlobInfo{ + { + Digest: "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f", + Size: 32654, + MediaType: imgspecv1.MediaTypeImageLayerGzip, + CompressionOperation: types.Compress, + CompressionAlgorithm: &compression.Zstd, + }, + }) + assert.Nil(t, err) + + updatedManifestBytes, err := manifest.Serialize() + assert.Nil(t, err) + + bytes, err = ioutil.ReadFile("fixtures/ociv1.nondistributable.zstd.manifest.json") + assert.Nil(t, err) + + expectedManifest, err := OCI1FromManifest(bytes) + assert.Nil(t, err) + + expectedManifestBytes, err := expectedManifest.Serialize() + assert.Nil(t, err) + + assert.Equal(t, string(expectedManifestBytes), string(updatedManifestBytes)) +} + +func TestUpdateLayerInfosOCINondistributableGzipToUncompressed(t *testing.T) { + bytes, err := ioutil.ReadFile("fixtures/ociv1.nondistributable.gzip.manifest.json") + assert.Nil(t, err) + + manifest, err := OCI1FromManifest(bytes) + assert.Nil(t, err) + + err = manifest.UpdateLayerInfos([]types.BlobInfo{ + { + Digest: "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f", + Size: 32654, + MediaType: imgspecv1.MediaTypeImageLayerGzip, + CompressionOperation: types.Decompress, + }, + }) + assert.Nil(t, err) + + updatedManifestBytes, err := manifest.Serialize() + assert.Nil(t, err) + + bytes, err = ioutil.ReadFile("fixtures/ociv1.nondistributable.manifest.json") + assert.Nil(t, err) + + expectedManifest, err := OCI1FromManifest(bytes) + assert.Nil(t, err) + + expectedManifestBytes, err := expectedManifest.Serialize() + assert.Nil(t, err) + + assert.Equal(t, string(expectedManifestBytes), string(updatedManifestBytes)) +} diff --git a/pkg/compression/compression.go b/pkg/compression/compression.go index b42151cffc..267868c6ab 100644 --- a/pkg/compression/compression.go +++ b/pkg/compression/compression.go @@ -13,6 +13,46 @@ import ( "github.com/ulikunitz/xz" ) +// Algorithm is a compression algorithm that can be used for CompressStream. +type Algorithm struct { + name string + prefix []byte + decompressor DecompressorFunc + compressor compressorFunc +} + +var ( + // Gzip compression. + Gzip = Algorithm{"gzip", []byte{0x1F, 0x8B, 0x08}, GzipDecompressor, gzipCompressor} + // Bzip2 compression. + Bzip2 = Algorithm{"bzip2", []byte{0x42, 0x5A, 0x68}, Bzip2Decompressor, bzip2Compressor} + // Xz compression. + Xz = Algorithm{"Xz", []byte{0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00}, XzDecompressor, xzCompressor} + // Zstd compression. + Zstd = Algorithm{"zstd", []byte{0x28, 0xb5, 0x2f, 0xfd}, ZstdDecompressor, zstdCompressor} + + compressionAlgorithms = map[string]Algorithm{ + Gzip.name: Gzip, + Bzip2.name: Bzip2, + Xz.name: Xz, + Zstd.name: Zstd, + } +) + +// Name returns the name for the compression algorithm. +func (c Algorithm) Name() string { + return c.name +} + +// AlgorithmByName returns the compressor by its name +func AlgorithmByName(name string) (Algorithm, error) { + algorithm, ok := compressionAlgorithms[name] + if ok { + return algorithm, nil + } + return Algorithm{}, fmt.Errorf("cannot find compressor for %q", name) +} + // DecompressorFunc returns the decompressed stream, given a compressed stream. // The caller must call Close() on the decompressed stream (even if the compressed input stream does not need closing!). type DecompressorFunc func(io.Reader) (io.ReadCloser, error) @@ -58,37 +98,6 @@ func xzCompressor(r io.Writer, level *int) (io.WriteCloser, error) { return xz.NewWriter(r) } -// Algorithm is a compression algorithm that can be used for CompressStream. -type Algorithm struct { - name string - prefix []byte - decompressor DecompressorFunc - compressor compressorFunc -} - -// Name returns the name for the compression algorithm. -func (c Algorithm) Name() string { - return c.name -} - -// compressionAlgos is an internal implementation detail of DetectCompression -var compressionAlgos = []Algorithm{ - {"gzip", []byte{0x1F, 0x8B, 0x08}, GzipDecompressor, gzipCompressor}, // gzip (RFC 1952) - {"bzip2", []byte{0x42, 0x5A, 0x68}, Bzip2Decompressor, bzip2Compressor}, // bzip2 (decompress.c:BZ2_decompress) - {"xz", []byte{0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00}, XzDecompressor, xzCompressor}, // xz (/usr/share/doc/xz/xz-file-format.txt) - {"zstd", []byte{0x28, 0xb5, 0x2f, 0xfd}, ZstdDecompressor, zstdCompressor}, // zstd (http://www.zstd.net) -} - -// AlgorithmByName returns the compressor by its name -func AlgorithmByName(name string) (Algorithm, error) { - for _, c := range compressionAlgos { - if c.name == name { - return c, nil - } - } - return Algorithm{}, fmt.Errorf("cannot find compressor for %q", name) -} - // CompressStream returns the compressor by its name func CompressStream(dest io.Writer, algo Algorithm, level *int) (io.WriteCloser, error) { return algo.compressor(dest, level) @@ -108,7 +117,7 @@ func DetectCompressionFormat(input io.Reader) (Algorithm, DecompressorFunc, io.R var retAlgo Algorithm var decompressor DecompressorFunc - for _, algo := range compressionAlgos { + for _, algo := range compressionAlgorithms { if bytes.HasPrefix(buffer[:n], algo.prefix) { logrus.Debugf("Detected compression format %s", algo.name) retAlgo = algo diff --git a/storage/storage_image.go b/storage/storage_image.go index 946a85f7b1..7d8860a508 100644 --- a/storage/storage_image.go +++ b/storage/storage_image.go @@ -345,9 +345,9 @@ func (s *storageImageDestination) Close() error { } func (s *storageImageDestination) DesiredLayerCompression() types.LayerCompression { - // We ultimately have to decompress layers to populate trees on disk, - // so callers shouldn't bother compressing them before handing them to - // us, if they're not already compressed. + // We ultimately have to decompress layers to populate trees on disk + // and need to explicitly ask for it here, so that the layers' MIME + // types can be set accordingly. return types.PreserveOriginal } diff --git a/types/types.go b/types/types.go index b94af8dccb..0d2fb7d86b 100644 --- a/types/types.go +++ b/types/types.go @@ -8,7 +8,7 @@ import ( "github.com/containers/image/docker/reference" "github.com/containers/image/pkg/compression" "github.com/opencontainers/go-digest" - "github.com/opencontainers/image-spec/specs-go/v1" + v1 "github.com/opencontainers/image-spec/specs-go/v1" ) // ImageTransport is a top-level namespace for ways to to store/load an image. @@ -91,6 +91,19 @@ type ImageReference interface { DeleteImage(ctx context.Context, sys *SystemContext) error } +// LayerCompression indicates if layers must be compressed, decompressed or preserved +type LayerCompression int + +const ( + // PreserveOriginal indicates the layer must be preserved, ie + // no compression or decompression. + PreserveOriginal LayerCompression = iota + // Decompress indicates the layer must be decompressed + Decompress + // Compress indicates the layer must be compressed + Compress +) + // BlobInfo collects known information about a blob (layer/config). // In some situations, some fields may be unknown, in others they may be mandatory; documenting an “unknown” value here does not override that. type BlobInfo struct { @@ -99,6 +112,14 @@ type BlobInfo struct { URLs []string Annotations map[string]string MediaType string + // CompressionOperation is used in Image.UpdateLayerInfos to instruct + // whether the original layer should be preserved or (de)compressed. The + // field defaults to preserve the original layer. + CompressionOperation LayerCompression + // CompressionAlgorithm is used in Image.UpdateLayerInfos to set the correct + // MIME type for compressed layers (e.g., gzip or zstd). This field MUST be + // set when `CompressionOperation == Compress`. + CompressionAlgorithm *compression.Algorithm } // BICTransportScope encapsulates transport-dependent representation of a “scope” where blobs are or are not present. @@ -212,19 +233,6 @@ type ImageSource interface { LayerInfosForCopy(ctx context.Context) ([]BlobInfo, error) } -// LayerCompression indicates if layers must be compressed, decompressed or preserved -type LayerCompression int - -const ( - // PreserveOriginal indicates the layer must be preserved, ie - // no compression or decompression. - PreserveOriginal LayerCompression = iota - // Decompress indicates the layer must be decompressed - Decompress - // Compress indicates the layer must be compressed - Compress -) - // ImageDestination is a service, possibly remote (= slow), to store components of a single image. // // There is a specific required order for some of the calls: