Skip to content

Commit

Permalink
Merge pull request #1896 from flouthoc/edit-instances
Browse files Browse the repository at this point in the history 
manifest: prepare internal `EditInstances`
  • Loading branch information
@mtrmac
mtrmac authored Jun 1, 2023
2 parents e14c1c5 + 091e672 commit e8c2d91
Show file tree
Hide file tree
Showing 5 changed files with 280 additions and 29 deletions.
81 changes: 66 additions & 15 deletions internal/manifest/docker_schema2_list.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,15 @@ func (list *Schema2ListPublic) MIMEType() string {
return list.MediaType
}

func (list *Schema2ListPublic) descriptorIndex(instanceDigest digest.Digest) int {
for i, m := range list.Manifests {
if m.Digest == instanceDigest {
return i
}
}
return -1
}

// Instances returns a slice of digests of the manifests that this list knows of.
func (list *Schema2ListPublic) Instances() []digest.Digest {
results := make([]digest.Digest, len(list.Manifests))
Expand All @@ -69,27 +78,69 @@ func (list *Schema2ListPublic) Instance(instanceDigest digest.Digest) (ListUpdat

// UpdateInstances updates the sizes, digests, and media types of the manifests
// which the list catalogs.
func (list *Schema2ListPublic) UpdateInstances(updates []ListUpdate) error {
if len(updates) != len(list.Manifests) {
return fmt.Errorf("incorrect number of update entries passed to Schema2List.UpdateInstances: expected %d, got %d", len(list.Manifests), len(updates))
func (index *Schema2ListPublic) UpdateInstances(updates []ListUpdate) error {
editInstances := []ListEdit{}
for i, instance := range updates {
editInstances = append(editInstances, ListEdit{
UpdateOldDigest: index.Manifests[i].Digest,
UpdateDigest: instance.Digest,
UpdateSize: instance.Size,
UpdateMediaType: instance.MediaType,
ListOperation: ListOpUpdate})
}
for i := range updates {
if err := updates[i].Digest.Validate(); err != nil {
return fmt.Errorf("update %d of %d passed to Schema2List.UpdateInstances contained an invalid digest: %w", i+1, len(updates), err)
}
list.Manifests[i].Digest = updates[i].Digest
if updates[i].Size < 0 {
return fmt.Errorf("update %d of %d passed to Schema2List.UpdateInstances had an invalid size (%d)", i+1, len(updates), updates[i].Size)
}
list.Manifests[i].Size = updates[i].Size
if updates[i].MediaType == "" {
return fmt.Errorf("update %d of %d passed to Schema2List.UpdateInstances had no media type (was %q)", i+1, len(updates), list.Manifests[i].MediaType)
return index.editInstances(editInstances)
}

func (index *Schema2ListPublic) editInstances(editInstances []ListEdit) error {
addedEntries := []Schema2ManifestDescriptor{}
for i, editInstance := range editInstances {
switch editInstance.ListOperation {
case ListOpUpdate:
if err := editInstance.UpdateOldDigest.Validate(); err != nil {
return fmt.Errorf("Schema2List.EditInstances: Attempting to update %s which is an invalid digest: %w", editInstance.UpdateOldDigest, err)
}
if err := editInstance.UpdateDigest.Validate(); err != nil {
return fmt.Errorf("Schema2List.EditInstances: Modified digest %s is an invalid digest: %w", editInstance.UpdateDigest, err)
}
targetIndex := index.descriptorIndex(editInstance.UpdateOldDigest)
if targetIndex < 0 {
return fmt.Errorf("Schema2List.EditInstances: Attempting to update %s which is an invalid digest", editInstance.UpdateOldDigest)
}
index.Manifests[targetIndex].Digest = editInstance.UpdateDigest
if editInstance.UpdateSize < 0 {
return fmt.Errorf("update %d of %d passed to Schema2List.UpdateInstances had an invalid size (%d)", i+1, len(editInstances), editInstance.UpdateSize)
}
index.Manifests[targetIndex].Size = editInstance.UpdateSize
if editInstance.UpdateMediaType == "" {
return fmt.Errorf("update %d of %d passed to Schema2List.UpdateInstances had no media type (was %q)", i+1, len(editInstances), index.Manifests[i].MediaType)
}
index.Manifests[targetIndex].MediaType = editInstance.UpdateMediaType
case ListOpAdd:
addInstance := Schema2ManifestDescriptor{
Schema2Descriptor{Digest: editInstance.AddDigest, Size: editInstance.AddSize, MediaType: editInstance.AddMediaType},
Schema2PlatformSpec{
OS: editInstance.AddPlatform.OS,
Architecture: editInstance.AddPlatform.Architecture,
OSVersion: editInstance.AddPlatform.OSVersion,
OSFeatures: editInstance.AddPlatform.OSFeatures,
Variant: editInstance.AddPlatform.Variant,
},
}
addedEntries = append(addedEntries, addInstance)
default:
return fmt.Errorf("internal error: invalid operation: %d", editInstance.ListOperation)
}
list.Manifests[i].MediaType = updates[i].MediaType
}
if len(addedEntries) != 0 {
index.Manifests = append(index.Manifests, addedEntries...)
}
return nil
}

func (index *Schema2List) EditInstances(editInstances []ListEdit) error {
return index.editInstances(editInstances)
}

func (list *Schema2ListPublic) ChooseInstanceByCompression(ctx *types.SystemContext, preferGzip types.OptionalBool) (digest.Digest, error) {
// ChooseInstanceByCompression is same as ChooseInstance for schema2 manifest list.
return list.ChooseInstance(ctx)
Expand Down
57 changes: 57 additions & 0 deletions internal/manifest/docker_schema2_list_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@ import (
"path/filepath"
"testing"

"github.com/opencontainers/go-digest"
imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

Expand All @@ -27,6 +30,60 @@ func TestSchema2ListPublicFromManifest(t *testing.T) {
testValidManifestWithExtraFieldsIsRejected(t, parser, validManifest, []string{"config", "fsLayers", "history", "layers"})
}

func TestSchema2ListEditInstances(t *testing.T) {
validManifest, err := os.ReadFile(filepath.Join("testdata", "v2list.manifest.json"))
require.NoError(t, err)
list, err := ListFromBlob(validManifest, GuessMIMEType(validManifest))
require.NoError(t, err)

expectedDigests := list.Instances()
editInstances := []ListEdit{}
editInstances = append(editInstances, ListEdit{
UpdateOldDigest: list.Instances()[0],
UpdateDigest: "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
UpdateSize: 32,
UpdateMediaType: "something",
ListOperation: ListOpUpdate})
err = list.EditInstances(editInstances)
require.NoError(t, err)

expectedDigests[0] = editInstances[0].UpdateDigest
// order of old elements must remain same.
assert.Equal(t, list.Instances(), expectedDigests)

instance, err := list.Instance(list.Instances()[0])
require.NoError(t, err)
assert.Equal(t, "something", instance.MediaType)
assert.Equal(t, int64(32), instance.Size)

// Create a fresh list
list, err = ListFromBlob(validManifest, GuessMIMEType(validManifest))
require.NoError(t, err)
originalListOrder := list.Instances()

editInstances = []ListEdit{}
editInstances = append(editInstances, ListEdit{
AddDigest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
AddSize: 32,
AddMediaType: "application/vnd.oci.image.manifest.v1+json",
AddPlatform: &imgspecv1.Platform{Architecture: "amd64", OS: "linux", OSFeatures: []string{"sse4"}},
ListOperation: ListOpAdd})
editInstances = append(editInstances, ListEdit{
AddDigest: "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc",
AddSize: 32,
AddMediaType: "application/vnd.oci.image.manifest.v1+json",
AddPlatform: &imgspecv1.Platform{Architecture: "amd64", OS: "linux", OSFeatures: []string{"sse4"}},
ListOperation: ListOpAdd})
err = list.EditInstances(editInstances)
require.NoError(t, err)

// Add new elements to the end of old list to maintain order
originalListOrder = append(originalListOrder, digest.Digest("sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"))
originalListOrder = append(originalListOrder, digest.Digest("sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"))
// Verify order
assert.Equal(t, list.Instances(), originalListOrder)
}

func TestSchema2ListFromManifest(t *testing.T) {
validManifest, err := os.ReadFile(filepath.Join("testdata", "v2list.manifest.json"))
require.NoError(t, err)
Expand Down
30 changes: 30 additions & 0 deletions internal/manifest/list.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,10 @@ type List interface {
// SystemContext ( or for the current platform if the SystemContext doesn't specify any detail ) and preferGzip for compression which
// when configured to OptionalBoolTrue and chooses best available compression when it is OptionalBoolFalse or left OptionalBoolUndefined.
ChooseInstanceByCompression(ctx *types.SystemContext, preferGzip types.OptionalBool) (digest.Digest, error)
// Edit information about the list's instances. Contains Slice of ListEdit where each element
// is responsible for either Modifying or Adding a new instance to the Manifest. Operation is
// selected on the basis of configured ListOperation field.
EditInstances([]ListEdit) error
}

// ListUpdate includes the fields which a List's UpdateInstances() method will modify.
Expand All @@ -65,6 +69,32 @@ type ListUpdate struct {
MediaType string
}

type ListOp int

const (
listOpInvalid ListOp = iota
ListOpAdd
ListOpUpdate
)

// ListEdit includes the fields which a List's EditInstances() method will modify.
type ListEdit struct {
ListOperation ListOp

// if Op == ListEditUpdate (basically the previous UpdateInstances). All fields must be set.
UpdateOldDigest digest.Digest
UpdateDigest digest.Digest
UpdateSize int64
UpdateMediaType string

// If Op = ListEditAdd. All fields must be set.
AddDigest digest.Digest
AddSize int64
AddMediaType string
AddPlatform *imgspecv1.Platform
AddAnnotations map[string]string
}

// ListPublicFromBlob parses a list of manifests.
// This is publicly visible as c/image/manifest.ListFromBlob.
func ListPublicFromBlob(manifest []byte, manifestMIMEType string) (ListPublic, error) {
Expand Down
78 changes: 64 additions & 14 deletions internal/manifest/oci_index.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"fmt"
"math"
"runtime"
"sort"

platform "github.com/containers/image/v5/internal/pkg/platform"
"github.com/containers/image/v5/types"
Expand Down Expand Up @@ -38,6 +39,15 @@ func (index *OCI1IndexPublic) MIMEType() string {
return imgspecv1.MediaTypeImageIndex
}

func (index *OCI1IndexPublic) descriptorIndex(instanceDigest digest.Digest) int {
for i, m := range index.Manifests {
if m.Digest == instanceDigest {
return i
}
}
return -1
}

// Instances returns a slice of digests of the manifests that this index knows of.
func (index *OCI1IndexPublic) Instances() []digest.Digest {
results := make([]digest.Digest, len(index.Manifests))
Expand All @@ -64,26 +74,66 @@ func (index *OCI1IndexPublic) Instance(instanceDigest digest.Digest) (ListUpdate
// UpdateInstances updates the sizes, digests, and media types of the manifests
// which the list catalogs.
func (index *OCI1IndexPublic) UpdateInstances(updates []ListUpdate) error {
if len(updates) != len(index.Manifests) {
return fmt.Errorf("incorrect number of update entries passed to OCI1Index.UpdateInstances: expected %d, got %d", len(index.Manifests), len(updates))
editInstances := []ListEdit{}
for i, instance := range updates {
editInstances = append(editInstances, ListEdit{
UpdateOldDigest: index.Manifests[i].Digest,
UpdateDigest: instance.Digest,
UpdateSize: instance.Size,
UpdateMediaType: instance.MediaType,
ListOperation: ListOpUpdate})
}
for i := range updates {
if err := updates[i].Digest.Validate(); err != nil {
return fmt.Errorf("update %d of %d passed to OCI1Index.UpdateInstances contained an invalid digest: %w", i+1, len(updates), err)
}
index.Manifests[i].Digest = updates[i].Digest
if updates[i].Size < 0 {
return fmt.Errorf("update %d of %d passed to OCI1Index.UpdateInstances had an invalid size (%d)", i+1, len(updates), updates[i].Size)
}
index.Manifests[i].Size = updates[i].Size
if updates[i].MediaType == "" {
return fmt.Errorf("update %d of %d passed to OCI1Index.UpdateInstances had no media type (was %q)", i+1, len(updates), index.Manifests[i].MediaType)
return index.editInstances(editInstances)
}

func (index *OCI1IndexPublic) editInstances(editInstances []ListEdit) error {
addedEntries := []imgspecv1.Descriptor{}
for i, editInstance := range editInstances {
switch editInstance.ListOperation {
case ListOpUpdate:
if err := editInstance.UpdateOldDigest.Validate(); err != nil {
return fmt.Errorf("OCI1Index.EditInstances: Attempting to update %s which is an invalid digest: %w", editInstance.UpdateOldDigest, err)
}
if err := editInstance.UpdateDigest.Validate(); err != nil {
return fmt.Errorf("OCI1Index.EditInstances: Modified digest %s is an invalid digest: %w", editInstance.UpdateDigest, err)
}
targetIndex := index.descriptorIndex(editInstance.UpdateOldDigest)
if targetIndex < 0 {
return fmt.Errorf("OCI1Index.EditInstances: Attempting to update %s which is an invalid digest", editInstance.UpdateOldDigest)
}
index.Manifests[targetIndex].Digest = editInstance.UpdateDigest
if editInstance.UpdateSize < 0 {
return fmt.Errorf("update %d of %d passed to OCI1Index.UpdateInstances had an invalid size (%d)", i+1, len(editInstances), editInstance.UpdateSize)
}
index.Manifests[targetIndex].Size = editInstance.UpdateSize
if editInstance.UpdateMediaType == "" {
return fmt.Errorf("update %d of %d passed to OCI1Index.UpdateInstances had no media type (was %q)", i+1, len(editInstances), index.Manifests[i].MediaType)
}
index.Manifests[targetIndex].MediaType = editInstance.UpdateMediaType
case ListOpAdd:
addedEntries = append(addedEntries, imgspecv1.Descriptor{
MediaType: editInstance.AddMediaType,
Size: editInstance.AddSize,
Digest: editInstance.AddDigest,
Platform: editInstance.AddPlatform,
Annotations: editInstance.AddAnnotations})
default:
return fmt.Errorf("internal error: invalid operation: %d", editInstance.ListOperation)
}
index.Manifests[i].MediaType = updates[i].MediaType
}
if len(addedEntries) != 0 {
index.Manifests = append(index.Manifests, addedEntries...)
sort.SliceStable(index.Manifests, func(i, j int) bool {
return !instanceIsZstd(index.Manifests[i]) && instanceIsZstd(index.Manifests[j])
})
}
return nil
}

func (index *OCI1Index) EditInstances(editInstances []ListEdit) error {
return index.editInstances(editInstances)
}

// instanceIsZstd returns true if instance is a zstd instance otherwise false.
func instanceIsZstd(manifest imgspecv1.Descriptor) bool {
if value, ok := manifest.Annotations[OCI1InstanceAnnotationCompressionZSTD]; ok && value == "true" {
Expand Down
63 changes: 63 additions & 0 deletions internal/manifest/oci_index_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (

"github.com/containers/image/v5/types"
"github.com/opencontainers/go-digest"
imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
Expand Down Expand Up @@ -50,6 +51,68 @@ func TestOCI1IndexFromManifest(t *testing.T) {
testValidManifestWithExtraFieldsIsRejected(t, parser, validManifest, []string{"config", "fsLayers", "history", "layers"})
}

func TestOCI1EditInstances(t *testing.T) {
validManifest, err := os.ReadFile(filepath.Join("testdata", "ociv1.image.index.json"))
require.NoError(t, err)
list, err := ListFromBlob(validManifest, GuessMIMEType(validManifest))
require.NoError(t, err)

expectedDigests := list.Instances()
editInstances := []ListEdit{}
editInstances = append(editInstances, ListEdit{
UpdateOldDigest: list.Instances()[0],
UpdateDigest: "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
UpdateSize: 32,
UpdateMediaType: "something",
ListOperation: ListOpUpdate})
err = list.EditInstances(editInstances)
require.NoError(t, err)

expectedDigests[0] = editInstances[0].UpdateDigest
// order of old elements must remain same.
assert.Equal(t, list.Instances(), expectedDigests)

instance, err := list.Instance(list.Instances()[0])
require.NoError(t, err)
assert.Equal(t, "something", instance.MediaType)
assert.Equal(t, int64(32), instance.Size)

// Create a fresh list
list, err = ListFromBlob(validManifest, GuessMIMEType(validManifest))
require.NoError(t, err)

// Verfiy correct zstd sorting
editInstances = []ListEdit{}
annotations := map[string]string{"io.github.containers.compression.zstd": "true"}
// without zstd
editInstances = append(editInstances, ListEdit{
AddDigest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
AddSize: 32,
AddMediaType: "application/vnd.oci.image.manifest.v1+json",
AddPlatform: &imgspecv1.Platform{Architecture: "amd64", OS: "linux", OSFeatures: []string{"sse4"}},
ListOperation: ListOpAdd})
// with zstd
editInstances = append(editInstances, ListEdit{
AddDigest: "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee",
AddSize: 32,
AddMediaType: "application/vnd.oci.image.manifest.v1+json",
AddPlatform: &imgspecv1.Platform{Architecture: "amd64", OS: "linux", OSFeatures: []string{"sse4"}},
AddAnnotations: annotations,
ListOperation: ListOpAdd})
// without zstd
editInstances = append(editInstances, ListEdit{
AddDigest: "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc",
AddSize: 32,
AddMediaType: "application/vnd.oci.image.manifest.v1+json",
AddPlatform: &imgspecv1.Platform{Architecture: "amd64", OS: "linux", OSFeatures: []string{"sse4"}},
ListOperation: ListOpAdd})
err = list.EditInstances(editInstances)
require.NoError(t, err)

// Zstd should be kept on lowest priority as compared to the default gzip ones and order of prior elements must be preserved.
assert.Equal(t, list.Instances(), []digest.Digest{digest.Digest("sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f"), digest.Digest("sha256:5b0bcabd1ed22e9fb1310cf6c2dec7cdef19f0ad69efa1f392e94a4333501270"), digest.Digest("sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), digest.Digest("sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"), digest.Digest("sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee")})
}

func TestOCI1IndexChooseInstanceByCompression(t *testing.T) {
type expectedMatch struct {
arch, variant string
Expand Down

0 comments on commit e8c2d91

Please sign in to comment.