copy: set media types

When copying an image, record the compression in the BlobInfo and use the information when updating the manifest's layer infos to set the layers' media types correctly. Note that consumers of the containers/image library need to update opencontainers/image-spec to commit 775207bd45b6cb8153ce218cc59351799217451f. Fixes: github.com/containers/podman/issues/2013 Fixes: github.com/containers/buildah/issues/1589 Signed-off-by: Valentin Rothberg <[email protected]>
containers · Aug 27, 2019 · 29253fb · 29253fb
1 parent 713c8d4
commit 29253fb
Show file tree

Hide file tree

Showing 31 changed files with 1,359 additions and 60 deletions.
diff --git a/copy/copy.go b/copy/copy.go
@@ -911,6 +911,12 @@ func (c *copier) copyBlobFromStream(ctx context.Context, srcStream io.Reader, sr
 		return types.BlobInfo{}, errors.Wrap(err, "Error writing blob")
 	}
 
+	uploadedInfo.CompressionOperation = compressionOperation
+	// If we can modify the layer's blob, set the desired algorithm for it to be set in the manifest.
+	if canModifyBlob && !isConfig {
+		uploadedInfo.CompressionAlgorithm = &desiredCompressionFormat
+	}
+
 	// This is fairly horrible: the writer from getOriginalLayerCopyWriter wants to consumer
 	// all of the input (to compute DiffIDs), even if dest.PutBlob does not need it.
 	// So, read everything from originalLayerReader, which will cause the rest to be

diff --git a/image/docker_schema2.go b/image/docker_schema2.go
@@ -6,6 +6,7 @@ import (
 	"crypto/sha256"
 	"encoding/hex"
 	"encoding/json"
+	"fmt"
 	"io/ioutil"
 	"strings"
 
@@ -207,12 +208,21 @@ func (m *manifestSchema2) convertToManifestOCI1(ctx context.Context) (types.Imag
 	layers := make([]imgspecv1.Descriptor, len(m.m.LayersDescriptors))
 	for idx := range layers {
 		layers[idx] = oci1DescriptorFromSchema2Descriptor(m.m.LayersDescriptors[idx])
-		if m.m.LayersDescriptors[idx].MediaType == manifest.DockerV2Schema2ForeignLayerMediaType {
+		switch m.m.LayersDescriptors[idx].MediaType {
+		case manifest.DockerV2Schema2ForeignLayerMediaType:
 			layers[idx].MediaType = imgspecv1.MediaTypeImageLayerNonDistributable
-		} else {
-			// we assume layers are gzip'ed because docker v2s2 only deals with
-			// gzip'ed layers. However, OCI has non-gzip'ed layers as well.
+		case manifest.DockerV2Schema2ForeignLayerMediaTypeGzip:
+			layers[idx].MediaType = imgspecv1.MediaTypeImageLayerNonDistributableGzip
+		case manifest.DockerV2Schema2ForeignLayerMediaTypeZstd:
+			layers[idx].MediaType = imgspecv1.MediaTypeImageLayerNonDistributableZstd
+		case manifest.DockerV2SchemaLayerMediaTypeUncompressed:
+			layers[idx].MediaType = imgspecv1.MediaTypeImageLayer
+		case manifest.DockerV2Schema2LayerMediaType:
 			layers[idx].MediaType = imgspecv1.MediaTypeImageLayerGzip
+		case manifest.DockerV2Schema2LayerMediaTypeZstd:
+			layers[idx].MediaType = imgspecv1.MediaTypeImageLayerZstd
+		default:
+			return nil, fmt.Errorf("Unknown media type during manifest conversion: %q", m.m.LayersDescriptors[idx].MediaType)
 		}
 	}
 

diff --git a/image/docker_schema2_test.go b/image/docker_schema2_test.go
@@ -520,6 +520,38 @@ func TestConvertToManifestOCI(t *testing.T) {
 	assert.Equal(t, byHand, converted)
 }
 
+func TestConvertToManifestOCIAllMediaTypes(t *testing.T) {
+	originalSrc := newSchema2ImageSource(t, "httpd-copy:latest")
+	original := manifestSchema2FromFixture(t, originalSrc, "schema2-all-media-types.json")
+	res, err := original.UpdatedImage(context.Background(), types.ManifestUpdateOptions{
+		ManifestMIMEType: imgspecv1.MediaTypeImageManifest,
+	})
+	require.NoError(t, err)
+
+	convertedJSON, mt, err := res.Manifest(context.Background())
+	require.NoError(t, err)
+	assert.Equal(t, imgspecv1.MediaTypeImageManifest, mt)
+
+	byHandJSON, err := ioutil.ReadFile("fixtures/oci1-all-media-types.json")
+	require.NoError(t, err)
+	var converted, byHand map[string]interface{}
+	err = json.Unmarshal(byHandJSON, &byHand)
+	require.NoError(t, err)
+	err = json.Unmarshal(convertedJSON, &converted)
+	require.NoError(t, err)
+	assert.Equal(t, byHand, converted)
+}
+
+func TestConvertToOCIWithInvalidMIMEType(t *testing.T) {
+	originalSrc := newSchema2ImageSource(t, "httpd-copy:latest")
+	original := manifestSchema2FromFixture(t, originalSrc, "schema2-invalid-media-type.json")
+	_, err := original.UpdatedImage(context.Background(), types.ManifestUpdateOptions{
+		ManifestMIMEType: imgspecv1.MediaTypeImageManifest,
+	})
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "Unknown media type during manifest conversion: ")
+}
+
 func TestConvertToManifestSchema1(t *testing.T) {
 	originalSrc := newSchema2ImageSource(t, "httpd-copy:latest")
 	original := manifestSchema2FromFixture(t, originalSrc, "schema2.json")

diff --git a/image/fixtures/oci1-all-media-types-to-schema2.json b/image/fixtures/oci1-all-media-types-to-schema2.json
@@ -0,0 +1,41 @@
+{
+    "schemaVersion": 2,
+    "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
+    "config": {
+       "mediaType": "application/vnd.docker.container.image.v1+json",
+       "size": 4651,
+       "digest": "sha256:a13a0762ab7bed51a1b49adec0a702b1cd99294fd460a025b465bcfb7b152745"
+    },
+    "layers": [
+        {
+            "mediaType": "application/vnd.docker.image.rootfs.diff.tar",
+            "size": 51354364,
+            "digest": "sha256:6a5a5368e0c2d3e5909184fa28ddfd56072e7ff3ee9a945876f7eee5896ef5bb"
+        },
+        {
+            "mediaType": "application/vnd.docker.image.rootfs.diff.tar.zstd",
+            "size": 150,
+            "digest": "sha256:1bbf5d58d24c47512e234a5623474acf65ae00d4d1414272a893204f44cc680c"
+        },
+        {
+            "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
+            "size": 152,
+            "digest": "sha256:2bbf5d58d24c47512e234a5623474acf65ae00d4d1414272a893204f44cc680c"
+        },
+        {
+            "mediaType": "application/vnd.docker.image.rootfs.foreign.diff.tar",
+            "size": 11739507,
+            "digest": "sha256:8f5dc8a4b12c307ac84de90cdd9a7f3915d1be04c9388868ca118831099c67a9"
+        },
+        {
+            "mediaType": "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip",
+            "size": 8841833,
+            "digest": "sha256:bbd6b22eb11afce63cc76f6bc41042d99f10d6024c96b655dafba930b8d25909"
+        },
+        {
+            "mediaType": "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip",
+            "size": 291,
+            "digest": "sha256:960e52ecf8200cbd84e70eb2ad8678f4367e50d14357021872c10fa3fc5935fa"
+        }
+    ]
+}
diff --git a/image/fixtures/oci1-all-media-types.json b/image/fixtures/oci1-all-media-types.json
@@ -0,0 +1,40 @@
+{
+    "schemaVersion": 2,
+    "config": {
+        "mediaType": "application/vnd.oci.image.config.v1+json",
+        "size": 4651,
+        "digest": "sha256:a13a0762ab7bed51a1b49adec0a702b1cd99294fd460a025b465bcfb7b152745"
+    },
+    "layers": [
+        {
+            "mediaType": "application/vnd.oci.image.layer.v1.tar",
+            "size": 51354364,
+            "digest": "sha256:6a5a5368e0c2d3e5909184fa28ddfd56072e7ff3ee9a945876f7eee5896ef5bb"
+        },
+        {
+            "mediaType": "application/vnd.oci.image.layer.v1.tar+zstd",
+            "size": 150,
+            "digest": "sha256:1bbf5d58d24c47512e234a5623474acf65ae00d4d1414272a893204f44cc680c"
+        },
+        {
+            "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip",
+            "size": 152,
+            "digest": "sha256:2bbf5d58d24c47512e234a5623474acf65ae00d4d1414272a893204f44cc680c"
+        },
+        {
+            "mediaType": "application/vnd.oci.image.layer.nondistributable.v1.tar",
+            "size": 11739507,
+            "digest": "sha256:8f5dc8a4b12c307ac84de90cdd9a7f3915d1be04c9388868ca118831099c67a9"
+        },
+        {
+            "mediaType": "application/vnd.oci.image.layer.nondistributable.v1.tar+gzip",
+            "size": 8841833,
+            "digest": "sha256:bbd6b22eb11afce63cc76f6bc41042d99f10d6024c96b655dafba930b8d25909"
+        },
+        {
+            "mediaType": "application/vnd.oci.image.layer.nondistributable.v1.tar+gzip",
+            "size": 291,
+            "digest": "sha256:960e52ecf8200cbd84e70eb2ad8678f4367e50d14357021872c10fa3fc5935fa"
+        }
+    ]
+}
diff --git a/image/fixtures/oci1-invalid-media-type.json b/image/fixtures/oci1-invalid-media-type.json
@@ -0,0 +1,15 @@
+{
+    "schemaVersion": 2,
+    "config": {
+        "mediaType": "application/vnd.oci.image.config.v1+json",
+        "size": 5940,
+        "digest": "sha256:9ca4bda0a6b3727a6ffcc43e981cad0f24e2ec79d338f6ba325b4dfd0756fb8f"
+    },
+    "layers": [
+        {
+            "mediaType": "application/vnd.oci.image.layer.v1.tar+invalid-suffix",
+            "size": 51354364,
+            "digest": "sha256:6a5a5368e0c2d3e5909184fa28ddfd56072e7ff3ee9a945876f7eee5896ef5bb"
+        }
+    ]
+}
diff --git a/image/fixtures/oci1-to-schema2.json b/image/fixtures/oci1-to-schema2.json
@@ -34,4 +34,4 @@
          "digest": "sha256:960e52ecf8200cbd84e70eb2ad8678f4367e50d14357021872c10fa3fc5935fa"
       }
    ]
-}
+}
diff --git a/image/fixtures/schema2-all-media-types.json b/image/fixtures/schema2-all-media-types.json
@@ -0,0 +1,41 @@
+{
+    "schemaVersion": 2,
+    "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
+    "config": {
+       "mediaType": "application/vnd.docker.container.image.v1+json",
+       "size": 4651,
+       "digest": "sha256:9ca4bda0a6b3727a6ffcc43e981cad0f24e2ec79d338f6ba325b4dfd0756fb8f"
+    },
+    "layers": [
+        {
+            "mediaType": "application/vnd.docker.image.rootfs.diff.tar",
+            "size": 51354364,
+            "digest": "sha256:6a5a5368e0c2d3e5909184fa28ddfd56072e7ff3ee9a945876f7eee5896ef5bb"
+        },
+        {
+            "mediaType": "application/vnd.docker.image.rootfs.diff.tar.zstd",
+            "size": 150,
+            "digest": "sha256:1bbf5d58d24c47512e234a5623474acf65ae00d4d1414272a893204f44cc680c"
+        },
+        {
+            "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
+            "size": 152,
+            "digest": "sha256:2bbf5d58d24c47512e234a5623474acf65ae00d4d1414272a893204f44cc680c"
+        },
+        {
+            "mediaType": "application/vnd.docker.image.rootfs.foreign.diff.tar",
+            "size": 11739507,
+            "digest": "sha256:8f5dc8a4b12c307ac84de90cdd9a7f3915d1be04c9388868ca118831099c67a9"
+        },
+        {
+            "mediaType": "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip",
+            "size": 8841833,
+            "digest": "sha256:bbd6b22eb11afce63cc76f6bc41042d99f10d6024c96b655dafba930b8d25909"
+        },
+        {
+            "mediaType": "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip",
+            "size": 291,
+            "digest": "sha256:960e52ecf8200cbd84e70eb2ad8678f4367e50d14357021872c10fa3fc5935fa"
+        }
+    ]
+}
diff --git a/image/fixtures/schema2-invalid-media-type.json b/image/fixtures/schema2-invalid-media-type.json
@@ -0,0 +1,36 @@
+{
+    "schemaVersion": 2,
+    "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
+    "config": {
+       "mediaType": "application/octet-stream",
+       "size": 5940,
+       "digest": "sha256:9ca4bda0a6b3727a6ffcc43e981cad0f24e2ec79d338f6ba325b4dfd0756fb8f"
+    },
+    "layers": [
+       {
+          "mediaType": "application/vnd.docker.image.rootfs.diff.tar.invalid",
+          "size": 51354364,
+          "digest": "sha256:6a5a5368e0c2d3e5909184fa28ddfd56072e7ff3ee9a945876f7eee5896ef5bb"
+       },
+       {
+          "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
+          "size": 150,
+          "digest": "sha256:1bbf5d58d24c47512e234a5623474acf65ae00d4d1414272a893204f44cc680c"
+       },
+       {
+          "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
+          "size": 11739507,
+          "digest": "sha256:8f5dc8a4b12c307ac84de90cdd9a7f3915d1be04c9388868ca118831099c67a9"
+       },
+       {
+          "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
+          "size": 8841833,
+          "digest": "sha256:bbd6b22eb11afce63cc76f6bc41042d99f10d6024c96b655dafba930b8d25909"
+       },
+       {
+          "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
+          "size": 291,
+          "digest": "sha256:960e52ecf8200cbd84e70eb2ad8678f4367e50d14357021872c10fa3fc5935fa"
+       }
+    ]
+ }
diff --git a/image/oci.go b/image/oci.go
@@ -3,6 +3,7 @@ package image
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"io/ioutil"
 
 	"github.com/containers/image/docker/reference"
@@ -187,7 +188,22 @@ func (m *manifestOCI1) convertToManifestSchema2() (types.Image, error) {
 	layers := make([]manifest.Schema2Descriptor, len(m.m.Layers))
 	for idx := range layers {
 		layers[idx] = schema2DescriptorFromOCI1Descriptor(m.m.Layers[idx])
-		layers[idx].MediaType = manifest.DockerV2Schema2LayerMediaType
+		switch layers[idx].MediaType {
+		case imgspecv1.MediaTypeImageLayerNonDistributable:
+			layers[idx].MediaType = manifest.DockerV2Schema2ForeignLayerMediaType
+		case imgspecv1.MediaTypeImageLayerNonDistributableGzip:
+			layers[idx].MediaType = manifest.DockerV2Schema2ForeignLayerMediaTypeGzip
+		case imgspecv1.MediaTypeImageLayerNonDistributableZstd:
+			layers[idx].MediaType = manifest.DockerV2Schema2ForeignLayerMediaTypeZstd
+		case imgspecv1.MediaTypeImageLayer:
+			layers[idx].MediaType = manifest.DockerV2SchemaLayerMediaTypeUncompressed
+		case imgspecv1.MediaTypeImageLayerGzip:
+			layers[idx].MediaType = manifest.DockerV2Schema2LayerMediaType
+		case imgspecv1.MediaTypeImageLayerZstd:
+			layers[idx].MediaType = manifest.DockerV2Schema2LayerMediaTypeZstd
+		default:
+			return nil, fmt.Errorf("Unknown media type during manifest conversion: %q", layers[idx].MediaType)
+		}
 	}
 
 	// Rather than copying the ConfigBlob now, we just pass m.src to the

diff --git a/image/oci_test.go b/image/oci_test.go
@@ -410,3 +410,35 @@ func TestConvertToManifestSchema2(t *testing.T) {
 
 	// FIXME? Test also the various failure cases, if only to see that we don't crash?
 }
+
+func TestConvertToManifestSchema2AllMediaTypes(t *testing.T) {
+	originalSrc := newOCI1ImageSource(t, "httpd-copy:latest")
+	original := manifestOCI1FromFixture(t, originalSrc, "oci1-all-media-types.json")
+	res, err := original.UpdatedImage(context.Background(), types.ManifestUpdateOptions{
+		ManifestMIMEType: manifest.DockerV2Schema2MediaType,
+	})
+	require.NoError(t, err)
+
+	convertedJSON, mt, err := res.Manifest(context.Background())
+	require.NoError(t, err)
+	assert.Equal(t, manifest.DockerV2Schema2MediaType, mt)
+
+	byHandJSON, err := ioutil.ReadFile("fixtures/oci1-all-media-types-to-schema2.json")
+	require.NoError(t, err)
+	var converted, byHand map[string]interface{}
+	err = json.Unmarshal(byHandJSON, &byHand)
+	require.NoError(t, err)
+	err = json.Unmarshal(convertedJSON, &converted)
+	require.NoError(t, err)
+	assert.Equal(t, byHand, converted)
+}
+
+func TestConvertToV2S2WithInvalidMIMEType(t *testing.T) {
+	originalSrc := newOCI1ImageSource(t, "httpd-copy:latest")
+	original := manifestOCI1FromFixture(t, originalSrc, "oci1-invalid-media-type.json")
+	_, err := original.UpdatedImage(context.Background(), types.ManifestUpdateOptions{
+		ManifestMIMEType: manifest.DockerV2Schema2MediaType,
+	})
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "Unknown media type during manifest conversion: ")
+}
diff --git a/image/sourced.go b/image/sourced.go
@@ -5,6 +5,7 @@ package image
 
 import (
 	"context"
+
 	"github.com/containers/image/types"
 )