diff --git a/Gopkg.lock b/Gopkg.lock index 62db1fba2..a352442c1 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -347,7 +347,7 @@ [[projects]] branch = "master" - digest = "1:61d8f5998565a7c687a03d7f93845975651a0e0a87a079b824a2905c125d9bd0" + digest = "1:2dc59dd841a809590c3b7d2366836870fb52d8cdd6c1943186fa05f8be4c8b9f" name = "github.com/docker/docker" packages = [ "api", @@ -387,6 +387,7 @@ "pkg/term", "pkg/term/windows", "pkg/urlutil", + "reference", "registry", "registry/resumable", ] @@ -1448,6 +1449,7 @@ "github.com/docker/cli/cli/context/kubernetes", "github.com/docker/cli/cli/context/store", "github.com/docker/cli/cli/flags", + "github.com/docker/cli/cli/streams", "github.com/docker/cli/opts", "github.com/docker/cli/templates", "github.com/docker/cnab-to-oci/relocation", @@ -1463,6 +1465,7 @@ "github.com/docker/docker/pkg/namesgenerator", "github.com/docker/docker/pkg/stringid", "github.com/docker/docker/pkg/term", + "github.com/docker/docker/reference", "github.com/docker/docker/registry", "github.com/docker/go-units", "github.com/docker/go/canonical/json", @@ -1491,6 +1494,7 @@ "gotest.tools/icmd", "k8s.io/api/core/v1", "k8s.io/apimachinery/pkg/apis/meta/v1", + "k8s.io/apimachinery/pkg/util/wait", "k8s.io/client-go/kubernetes/typed/core/v1", ] solver-name = "gps-cdcl" diff --git a/e2e/build_test.go b/e2e/build_test.go index 6aa71092b..79ff6f252 100644 --- a/e2e/build_test.go +++ b/e2e/build_test.go @@ -30,9 +30,16 @@ func TestBuild(t *testing.T) { cmd.Command = dockerCli.Command("app", "build", "--tag", "single:1.0.0", "--iidfile", iidfile, "-f", path.Join(testDir, "single.dockerapp"), testDir) icmd.RunCmd(cmd).Assert(t, icmd.Success) + _, err := os.Stat(iidfile) + assert.NilError(t, err) + bytes, err := ioutil.ReadFile(iidfile) + assert.NilError(t, err) + iid := string(bytes) + cfg := getDockerConfigDir(t, cmd) - f := path.Join(cfg, "app", "bundles", "docker.io", "library", "single", "_tags", "1.0.0", image.BundleFilename) + s := strings.Split(iid, ":") + f := path.Join(cfg, "app", "bundles", "contents", s[0], s[1], image.BundleFilename) bndl, err := image.FromFile(f) assert.NilError(t, err) @@ -47,11 +54,6 @@ func TestBuild(t *testing.T) { assert.Assert(t, img.Image == "" || strings.Contains(img.Image, "@sha256:")) } - _, err = os.Stat(iidfile) - assert.NilError(t, err) - bytes, err := ioutil.ReadFile(iidfile) - assert.NilError(t, err) - iid := string(bytes) actualID, err := store.FromAppImage(bndl) assert.NilError(t, err) assert.Equal(t, iid, fmt.Sprintf("sha256:%s", actualID.String())) @@ -67,31 +69,6 @@ func TestBuildMultiTag(t *testing.T) { tags := []string{"1.0.0", "latest"} cmd.Command = dockerCli.Command("app", "build", "--tag", "single:"+tags[0], "--tag", "single:"+tags[1], "--iidfile", iidfile, "-f", path.Join(testDir, "single.dockerapp"), testDir) icmd.RunCmd(cmd).Assert(t, icmd.Success) - - cfg := getDockerConfigDir(t, cmd) - - for _, tag := range tags { - f := path.Join(cfg, "app", "bundles", "docker.io", "library", "single", "_tags", tag, image.BundleFilename) - img, err := image.FromFile(f) - assert.NilError(t, err) - built := []string{img.InvocationImages[0].Digest, img.Images["web"].Digest, img.Images["worker"].Digest} - for _, ref := range built { - cmd.Command = dockerCli.Command("inspect", ref) - icmd.RunCmd(cmd).Assert(t, icmd.Success) - } - for _, img := range img.Images { - // Check all image not being built locally get a fixed reference - assert.Assert(t, img.Image == "" || strings.Contains(img.Image, "@sha256:")) - } - _, err = os.Stat(iidfile) - assert.NilError(t, err) - bytes, err := ioutil.ReadFile(iidfile) - assert.NilError(t, err) - iid := string(bytes) - actualID, err := store.FromAppImage(img) - assert.NilError(t, err) - assert.Equal(t, iid, fmt.Sprintf("sha256:%s", actualID.String())) - } }) } @@ -126,13 +103,13 @@ func TestBuildWithoutTag(t *testing.T) { cfg := getDockerConfigDir(t, cmd) - f := path.Join(cfg, "app", "bundles", "_ids") + f := path.Join(cfg, "app", "bundles", "contents", "sha256") infos, err := ioutil.ReadDir(f) assert.NilError(t, err) assert.Equal(t, len(infos), 1) id := infos[0].Name() - f = path.Join(cfg, "app", "bundles", "_ids", id, image.BundleFilename) + f = path.Join(cfg, "app", "bundles", "contents", "sha256", id, image.BundleFilename) data, err := ioutil.ReadFile(f) assert.NilError(t, err) var bndl bundle.Bundle @@ -157,13 +134,13 @@ func TestBuildWithArgs(t *testing.T) { cfg := getDockerConfigDir(t, cmd) - f := path.Join(cfg, "app", "bundles", "_ids") + f := path.Join(cfg, "app", "bundles", "contents", "sha256") infos, err := ioutil.ReadDir(f) assert.NilError(t, err) assert.Equal(t, len(infos), 1) id := infos[0].Name() - f = path.Join(cfg, "app", "bundles", "_ids", id, image.BundleFilename) + f = path.Join(cfg, "app", "bundles", "contents", "sha256", id, image.BundleFilename) data, err := ioutil.ReadFile(f) assert.NilError(t, err) var bndl bundle.Bundle diff --git a/e2e/compatibility_test.go b/e2e/compatibility_test.go index 5d73a7c6f..a73ae5b9f 100644 --- a/e2e/compatibility_test.go +++ b/e2e/compatibility_test.go @@ -13,6 +13,8 @@ import ( "testing" "time" + "github.com/opencontainers/go-digest" + "gotest.tools/assert" "k8s.io/apimachinery/pkg/util/wait" @@ -72,9 +74,13 @@ func TestBackwardsCompatibilityV1(t *testing.T) { data, err := ioutil.ReadFile(filepath.Join("testdata", "compatibility", "bundle-v0.9.0.json")) assert.NilError(t, err) // update bundle - bundleDir := filepath.Join(info.configDir, "app", "bundles", "docker.io", "library", "app-e2e", "_tags", "v0.9.0") + dg := digest.SHA256.FromBytes(data) + bundleDir := filepath.Join(info.configDir, "app", "bundles", "contents", dg.Algorithm().String(), dg.Encoded()) assert.NilError(t, os.MkdirAll(bundleDir, os.FileMode(0777))) assert.NilError(t, ioutil.WriteFile(filepath.Join(bundleDir, "bundle.json"), data, os.FileMode(0644))) + metadata := filepath.Join(info.configDir, "app", "bundles", "repositories.json") + json := fmt.Sprintf("{\"Repositories\":{\"app-e2e\":{\"app-e2e:v0.9.0\":\"%s\"}}}", dg.String()) + assert.NilError(t, ioutil.WriteFile(metadata, []byte(json), 0777)) // load images build with an old Docker App version assert.NilError(t, loadAndTagImage(info, info.tmpDir, "app-e2e:0.1.0-invoc", "https://github.com/docker/app-e2e/raw/master/images/v0.9.0/app-e2e-invoc.tar")) diff --git a/e2e/helper_test.go b/e2e/helper_test.go index 9127fee37..d489086a9 100644 --- a/e2e/helper_test.go +++ b/e2e/helper_test.go @@ -122,9 +122,14 @@ func runWithDindSwarmAndRegistry(t *testing.T, todo func(dindSwarmAndRegistryInf todo(runner) } -func build(t *testing.T, cmd icmd.Cmd, dockerCli dockerCliCommand, ref, path string) { - cmd.Command = dockerCli.Command("app", "build", "-t", ref, path) +func build(t *testing.T, cmd icmd.Cmd, dockerCli dockerCliCommand, ref, path string) string { + iidfile := fs.NewFile(t, "iid") + defer iidfile.Remove() + cmd.Command = dockerCli.Command("app", "build", "--iidfile", iidfile.Path(), "-t", ref, path) icmd.RunCmd(cmd).Assert(t, icmd.Success) + bytes, err := ioutil.ReadFile(iidfile.Path()) + assert.NilError(t, err) + return string(bytes) } // Container represents a docker container diff --git a/e2e/images_test.go b/e2e/images_test.go index af29a1108..d3114e775 100644 --- a/e2e/images_test.go +++ b/e2e/images_test.go @@ -148,7 +148,7 @@ Deleted: b-simple-app:latest`, cmd.Command = dockerCli.Command("app", "image", "rm", "b-simple-app") icmd.RunCmd(cmd).Assert(t, icmd.Expected{ ExitCode: 1, - Err: `b-simple-app:latest: reference not found`, + Err: `b-simple-app: reference not found`, }) expectedOutput := "REPOSITORY TAG APP IMAGE ID APP NAME \n" diff --git a/e2e/pushpull_test.go b/e2e/pushpull_test.go index c90f1678a..cd7b14c35 100644 --- a/e2e/pushpull_test.go +++ b/e2e/pushpull_test.go @@ -19,7 +19,7 @@ func TestPushUnknown(t *testing.T) { cmd.Command = dockerCli.Command("app", "push", "unknown") icmd.RunCmd(cmd).Assert(t, icmd.Expected{ ExitCode: 1, - Err: `could not push "unknown:latest": no such App image: failed to read bundle "docker.io/library/unknown:latest": unknown:latest: reference not found`, + Err: `could not push "unknown": unknown: reference not found`, }) }) @@ -27,7 +27,7 @@ func TestPushUnknown(t *testing.T) { cmd.Command = dockerCli.Command("app", "push", "@") icmd.RunCmd(cmd).Assert(t, icmd.Expected{ ExitCode: 1, - Err: `could not push "@": invalid reference format`, + Err: `could not push "@": could not parse "@" as a valid reference: invalid reference format`, }) }) } diff --git a/e2e/relocation_test.go b/e2e/relocation_test.go index da9cf742c..ca4b6ec20 100644 --- a/e2e/relocation_test.go +++ b/e2e/relocation_test.go @@ -13,34 +13,6 @@ import ( "gotest.tools/icmd" ) -func TestRelocationMapCreatedOnPull(t *testing.T) { - runWithDindSwarmAndRegistry(t, func(info dindSwarmAndRegistryInfo) { - cmd := info.configuredCmd - cfg := getDockerConfigDir(t, cmd) - - path := filepath.Join("testdata", "local") - ref := info.registryAddress + "/test/local:a-tag" - bundlePath := filepath.Join(cfg, "app", "bundles", strings.Replace(info.registryAddress, ":", "_", 1), "test", "local", "_tags", "a-tag") - - // Given a pushed application - build(t, cmd, dockerCli, ref, path) - cmd.Command = dockerCli.Command("app", "push", ref) - icmd.RunCmd(cmd).Assert(t, icmd.Success) - // And given application files are remove - assert.NilError(t, os.RemoveAll(bundlePath)) - _, err := os.Stat(filepath.Join(bundlePath, image.BundleFilename)) - assert.Assert(t, os.IsNotExist(err)) - - // When application is pulled - cmd.Command = dockerCli.Command("app", "pull", ref) - icmd.RunCmd(cmd).Assert(t, icmd.Success) - - // Then the relocation map should exist - _, err = os.Stat(filepath.Join(bundlePath, image.RelocationMapFilename)) - assert.NilError(t, err) - }) -} - func TestRelocationMapRun(t *testing.T) { runWithDindSwarmAndRegistry(t, func(info dindSwarmAndRegistryInfo) { cmd := info.configuredCmd @@ -82,7 +54,12 @@ func TestRelocationMapRun(t *testing.T) { t.Run("without-relocation-map", func(t *testing.T) { name := "test-relocation-map-run-without-relocation-map" // And given the relocation map is removed after the pull - assert.NilError(t, os.RemoveAll(filepath.Join(bundlePath, image.RelocationMapFilename))) + assert.NilError(t, filepath.Walk(filepath.Join(cfg, "app", "bundles", "contents"), func(path string, info os.FileInfo, err error) error { + if info.Name() == image.RelocationMapFilename { + os.Remove(path) + } + return nil + })) // Then the application cannot be run cmd.Command = dockerCli.Command("app", "run", "--name", name, ref) @@ -98,19 +75,13 @@ func TestPushPulledApplication(t *testing.T) { path := filepath.Join("testdata", "local") ref := info.registryAddress + "/test/local:a-tag" - bundlePath := filepath.Join(cfg, "app", "bundles", strings.Replace(info.registryAddress, ":", "_", 1), "test", "local", "_tags", "a-tag") // Given an application pushed on a registry build(t, cmd, dockerCli, ref, path) cmd.Command = dockerCli.Command("app", "push", ref) icmd.RunCmd(cmd).Assert(t, icmd.Success) // And given local images are removed - cmd.Command = dockerCli.Command("rmi", "web", "local:1.1.0-beta1-invoc", "worker") - icmd.RunCmd(cmd).Assert(t, icmd.Success) - // And given application files are remove - assert.NilError(t, os.RemoveAll(bundlePath)) - _, err := os.Stat(filepath.Join(bundlePath, image.BundleFilename)) - assert.Assert(t, os.IsNotExist(err)) + assert.NilError(t, os.RemoveAll(filepath.Join(cfg, "app", "bundles"))) // And given application is pulled from the registry cmd.Command = dockerCli.Command("app", "pull", ref) @@ -119,13 +90,6 @@ func TestPushPulledApplication(t *testing.T) { // Then the application can still be pushed cmd.Command = dockerCli.Command("app", "push", ref) icmd.RunCmd(cmd).Assert(t, icmd.Success) - - // If relocation map is removed - assert.NilError(t, os.RemoveAll(filepath.Join(bundlePath, image.RelocationMapFilename))) - - // Then the application cannot be pushed - cmd.Command = dockerCli.Command("app", "push", ref) - icmd.RunCmd(cmd).Assert(t, icmd.Expected{ExitCode: 1}) }) } diff --git a/internal/cnab/cnab.go b/internal/cnab/cnab.go index 1af6978ec..4418d9c62 100644 --- a/internal/cnab/cnab.go +++ b/internal/cnab/cnab.go @@ -123,7 +123,7 @@ func PullBundle(dockerCli command.Cli, imageStore appstore.ImageStore, tagRef re return nil, err } relocatedBundle := &image.AppImage{Bundle: bndl, RelocationMap: relocationMap} - if _, err := imageStore.Store(tagRef, relocatedBundle); err != nil { + if _, err := imageStore.Store(relocatedBundle, tagRef); err != nil { return nil, err } return relocatedBundle, nil diff --git a/internal/commands/image/list_test.go b/internal/commands/image/list_test.go index 26f6ce7a6..0fe55ea83 100644 --- a/internal/commands/image/list_test.go +++ b/internal/commands/image/list_test.go @@ -22,7 +22,7 @@ type imageStoreStubForListCmd struct { refList []reference.Reference } -func (b *imageStoreStubForListCmd) Store(ref reference.Reference, bndl *image.AppImage) (reference.Digested, error) { +func (b *imageStoreStubForListCmd) Store(bndl *image.AppImage, ref reference.Reference) (reference.Digested, error) { b.refMap[ref] = bndl b.refList = append(b.refList, ref) return store.FromAppImage(bndl) @@ -142,7 +142,7 @@ func testRunList(t *testing.T, refs []reference.Reference, bundles []image.AppIm refList: []reference.Reference{}, } for i, ref := range refs { - _, err = imageStore.Store(ref, &bundles[i]) + _, err = imageStore.Store(&bundles[i], ref) assert.NilError(t, err) } err = runList(dockerCli, options, imageStore) diff --git a/internal/commands/image/tag.go b/internal/commands/image/tag.go index 8fa157733..0925f7282 100644 --- a/internal/commands/image/tag.go +++ b/internal/commands/image/tag.go @@ -73,6 +73,6 @@ func storeBundle(bundle *image.AppImage, name string, imageStore store.ImageStor if err != nil { return err } - _, err = imageStore.Store(cnabRef, bundle) + _, err = imageStore.Store(bundle, cnabRef) return err } diff --git a/internal/commands/image/tag_test.go b/internal/commands/image/tag_test.go index 20cf59be4..acc95c794 100644 --- a/internal/commands/image/tag_test.go +++ b/internal/commands/image/tag_test.go @@ -28,7 +28,7 @@ type imageStoreStub struct { LookUpError error } -func (b *imageStoreStub) Store(ref reference.Reference, bndle *image.AppImage) (reference.Digested, error) { +func (b *imageStoreStub) Store(img *image.AppImage, ref reference.Reference) (reference.Digested, error) { defer func() { b.StoredError = nil }() diff --git a/internal/commands/push.go b/internal/commands/push.go index 721cbce9a..2e91c1071 100644 --- a/internal/commands/push.go +++ b/internal/commands/push.go @@ -59,17 +59,21 @@ func runPush(dockerCli command.Cli, name string) error { } // Get the bundle - ref, err := reference.ParseDockerRef(name) + ref, err := imageStore.LookUp(name) if err != nil { return errors.Wrapf(err, "could not push %q", name) } + named, ok := ref.(reference.Named) + if !ok { + return fmt.Errorf("could not push by ID. first tag your app image") + } - bndl, err := resolveReferenceAndBundle(imageStore, ref) + bndl, err := resolveReferenceAndBundle(imageStore, named) if err != nil { return err } - cnabRef := reference.TagNameOnly(ref) + cnabRef := reference.TagNameOnly(named) // Push the bundle return pushBundle(dockerCli, bndl, cnabRef) diff --git a/internal/packager/bundle.go b/internal/packager/bundle.go index 1d911d5d7..0723bc05f 100644 --- a/internal/packager/bundle.go +++ b/internal/packager/bundle.go @@ -82,7 +82,7 @@ func PersistInImageStore(ref reference.Reference, bndl *bundle.Bundle) (referenc if err != nil { return nil, err } - return imageStore.Store(ref, image.FromBundle(bndl)) + return imageStore.Store(image.FromBundle(bndl), ref) } func GetNamedTagged(tag string) (reference.NamedTagged, error) { diff --git a/internal/store/digest.go b/internal/store/digest.go index 2416b2561..5794ed7da 100644 --- a/internal/store/digest.go +++ b/internal/store/digest.go @@ -44,28 +44,28 @@ func StringToNamedRef(s string) (reference.Named, error) { func FromString(s string) (ID, error) { if ok := identifierRegexp.MatchString(s); !ok { - return ID{}, fmt.Errorf("could not parse %q as a valid reference", s) + return "", fmt.Errorf("could not parse %q as a valid reference", s) } + return fromID(s), nil +} + +func fromID(s string) ID { digest := digest.NewDigestFromEncoded(digest.SHA256, s) - return ID{digest}, nil + return ID(digest) } func FromAppImage(img *image.AppImage) (ID, error) { digest, err := ComputeDigest(img) - return ID{digest}, err + return ID(digest), err } // ID is an unique identifier for docker app image bundle, implementing reference.Reference -type ID struct { - digest digest.Digest -} - -var _ reference.Reference = ID{} +type ID digest.Digest func (id ID) String() string { - return id.digest.Encoded() + return id.Digest().Encoded() } func (id ID) Digest() digest.Digest { - return id.digest + return digest.Digest(id) } diff --git a/internal/store/digest_test.go b/internal/store/digest_test.go index 455a9ff01..21696429d 100644 --- a/internal/store/digest_test.go +++ b/internal/store/digest_test.go @@ -21,21 +21,14 @@ func Test_storeByDigest(t *testing.T) { assert.NilError(t, err) bndl := image.FromBundle(&bundle.Bundle{Name: "bundle-name"}) - ref := parseRefOrDie(t, "test/simple:1.0") - _, err = imageStore.Store(ref, bndl) + _, err = imageStore.Store(bndl, nil) assert.NilError(t, err) - _, err = os.Stat(dockerConfigDir.Join("app", "bundles", "docker.io", "test", "simple", "_tags", "1.0", image.BundleFilename)) - assert.NilError(t, err) - - _, err = imageStore.Store(nil, bndl) - assert.NilError(t, err) - - ids := dockerConfigDir.Join("app", "bundles", "_ids") + ids := dockerConfigDir.Join("app", "bundles", "contents", "sha256") infos, err := ioutil.ReadDir(ids) assert.NilError(t, err) assert.Equal(t, len(infos), 1) - _, err = os.Stat(dockerConfigDir.Join("app", "bundles", "_ids", infos[0].Name(), image.BundleFilename)) + _, err = os.Stat(dockerConfigDir.Join("app", "bundles", "contents", "sha256", infos[0].Name(), image.BundleFilename)) assert.NilError(t, err) } diff --git a/internal/store/image.go b/internal/store/image.go index c28a90057..147b8fb92 100644 --- a/internal/store/image.go +++ b/internal/store/image.go @@ -2,7 +2,6 @@ package store import ( "fmt" - "io" "os" "path/filepath" "sort" @@ -10,15 +9,16 @@ import ( "github.com/docker/app/internal/image" "github.com/docker/distribution/reference" - multierror "github.com/hashicorp/go-multierror" - digest "github.com/opencontainers/go-digest" + refstore "github.com/docker/docker/reference" + "github.com/hashicorp/go-multierror" + "github.com/opencontainers/go-digest" "github.com/pkg/errors" ) // type ImageStore interface { // Store do store the bundle with optional reference, and return it's unique ID - Store(ref reference.Reference, bndl *image.AppImage) (reference.Digested, error) + Store(img *image.AppImage, ref reference.Reference) (reference.Digested, error) Read(ref reference.Reference) (*image.AppImage, error) List() ([]reference.Reference, error) Remove(ref reference.Reference, force bool) error @@ -32,16 +32,23 @@ type referencesMap map[ID][]reference.Reference type imageStore struct { path string refsMap referencesMap + store refstore.Store } // NewImageStore creates a new bundle store with the given path and initializes it func NewImageStore(path string) (ImageStore, error) { + err := os.MkdirAll(filepath.Join(path, "contents", "sha256"), 0755) + if err != nil { + return nil, err + } + store, err := refstore.NewReferenceStore(filepath.Join(path, "repositories.json")) + if err != nil { + return nil, err + } imageStore := &imageStore{ path: path, refsMap: make(referencesMap), - } - if err := imageStore.scanAllBundles(); err != nil { - return nil, err + store: store, } return imageStore, nil } @@ -49,32 +56,19 @@ func NewImageStore(path string) (ImageStore, error) { // We store bundles either by image:tags, image:digest or by unique ID (actually, bundle's sha256). // // Within the bundle store, the file layout is -// -// \_ -// \_ _tags -// \_ -// \_ bundle.json -// \_ _digests -// \_ -// \_ -// \_ bundle.json -// _ids +// contents // \_ // \_ bundle.json +// \_ relocation-map.json +// repositories.json // managed by docker/reference // -func (b *imageStore) Store(ref reference.Reference, img *image.AppImage) (reference.Digested, error) { +func (b *imageStore) Store(img *image.AppImage, ref reference.Reference) (reference.Digested, error) { id, err := FromAppImage(img) if err != nil { return nil, errors.Wrapf(err, "failed to store bundle %q", ref) } - if ref == nil { - ref = id - } - dir, err := b.storePath(ref) - if err != nil { - return id, errors.Wrapf(err, "failed to store bundle %q", ref) - } + dir := b.storePath(id.Digest()) if err := os.MkdirAll(dir, 0755); err != nil { return id, errors.Wrapf(err, "failed to store bundle %q", ref) } @@ -83,25 +77,58 @@ func (b *imageStore) Store(ref reference.Reference, img *image.AppImage) (refere return id, errors.Wrapf(err, "failed to store app image %q", ref) } - b.refsMap.appendRef(id, ref) + if tag, ok := ref.(reference.NamedTagged); ok { + err = b.store.AddTag(reference.TagNameOnly(tag), id.Digest(), true) + if err != nil { + return nil, err + } + } + if digest, ok := ref.(reference.Canonical); ok { + err = b.store.AddDigest(digest, id.Digest(), true) + if err != nil { + return nil, err + } + } + return id, nil } func (b *imageStore) Read(ref reference.Reference) (*image.AppImage, error) { - paths, err := b.storePaths(ref) - if err != nil { - return nil, errors.Wrapf(err, "failed to read bundle %q", ref) + var dg digest.Digest + if id, ok := ref.(ID); ok { + dg = id.Digest() } - - return image.FromFile(filepath.Join(paths[0], image.BundleFilename)) + if named, ok := ref.(reference.Named); ok { + resolved, err := b.store.Get(reference.TagNameOnly(named)) + if err == refstore.ErrDoesNotExist { + return nil, unknownReference(ref.String()) + } + if err != nil { + return nil, err + } + dg = resolved + } + path := b.storePath(dg) + return image.FromFile(filepath.Join(path, image.BundleFilename)) } // Returns the list of all bundles present in the bundle store func (b *imageStore) List() ([]reference.Reference, error) { - var references []reference.Reference + ids, err := b.listIDs() + if err != nil { + return nil, err + } - for _, refAliases := range b.refsMap { - references = append(references, refAliases...) + references := []reference.Reference{} + for _, dg := range ids { + id := fromID(dg) + refs := b.store.References(id.Digest()) + for _, r := range refs { + references = append(references, r) + } + if len(refs) == 0 { + references = append(references, id) + } } sort.Slice(references, func(i, j int) bool { @@ -111,78 +138,66 @@ func (b *imageStore) List() ([]reference.Reference, error) { return references, nil } -// Remove removes a bundle from the bundle store. -func (b *imageStore) Remove(ref reference.Reference, force bool) error { - if id, ok := ref.(ID); ok { - refs := b.refsMap[id] - if len(refs) == 0 { - return fmt.Errorf("no such image %q", reference.FamiliarString(ref)) - } else if len(refs) > 1 { - if force { - var failures *multierror.Error - toDelete := append([]reference.Reference{}, refs...) - for _, r := range toDelete { - if err := b.doRemove(r); err != nil { - failures = multierror.Append(failures, err) - } - } - return failures.ErrorOrNil() - } - return fmt.Errorf("unable to delete %q - App is referenced in multiple repositories", reference.FamiliarString(ref)) - } - ref = refs[0] - } - return b.doRemove(ref) -} - -func (b *imageStore) doRemove(ref reference.Reference) error { - path, err := b.storePath(ref) +func (b *imageStore) listIDs() ([]string, error) { + f, err := os.Open(filepath.Join(b.path, "contents", "sha256")) if err != nil { - return err - } - if _, err := os.Stat(path); os.IsNotExist(err) { - return errors.New("no such image " + reference.FamiliarString(ref)) + return nil, err } - b.refsMap.removeRef(ref) - - if err := os.RemoveAll(path); err != nil { - return nil + defer f.Close() + ids, err := f.Readdirnames(-1) + if err != nil { + return nil, err } - return cleanupParentTree(path) + return ids, nil } -func cleanupParentTree(path string) error { - for { - path = filepath.Dir(path) - if empty, err := isEmpty(path); err != nil || !empty { +// Remove removes a bundle from the bundle store. +func (b *imageStore) Remove(ref reference.Reference, force bool) error { + if named, ok := ref.(reference.Named); ok { + named = reference.TagNameOnly(named) + id, err := b.store.Get(named) + if err != nil { + return err + } + _, err = b.store.Delete(named) + references := b.store.References(id) + if len(references) > 0 { return err } - if err := os.RemoveAll(path); err != nil { - return nil + // No tag left for ID, so also remove + ref = ID(id) + } + id := ref.(ID) + refs := b.store.References(id.Digest()) + if len(refs) > 1 && !force { + return fmt.Errorf("unable to delete %q - App is referenced in multiple repositories", reference.FamiliarString(ref)) + } + var failures *multierror.Error + for _, r := range refs { + if _, err := b.store.Delete(r); err != nil { + failures = multierror.Append(failures, err) } } -} - -func isEmpty(path string) (bool, error) { - f, err := os.Open(path) - if err != nil { - return false, err + if failures != nil { + return failures.ErrorOrNil() } - defer f.Close() - if _, err = f.Readdir(1); err == io.EOF { - // dir is empty - return true, nil + + path := b.storePath(id.Digest()) + _, err := os.Stat(b.storePath(id.Digest())) + if os.IsNotExist(err) { + return unknownReference(ref.String()) } - return false, nil + return os.RemoveAll(path) } func (b *imageStore) LookUp(refOrID string) (reference.Reference, error) { - ref, err := FromString(refOrID) + id, err := FromString(refOrID) if err == nil { - if _, found := b.refsMap[ref]; !found { + _, err := os.Stat(b.storePath(id.Digest())) + if os.IsNotExist(err) { return nil, unknownReference(refOrID) } - return ref, nil + return id, err } if isShortID(refOrID) { ref, err := b.matchShortID(refOrID) @@ -195,174 +210,46 @@ func (b *imageStore) LookUp(refOrID string) (reference.Reference, error) { return nil, err } if _, err = b.referenceToID(named); err != nil { + if err == refstore.ErrDoesNotExist { + return nil, unknownReference(refOrID) + } return nil, err } return named, nil } func (b *imageStore) matchShortID(shortID string) (reference.Reference, error) { - var found reference.Reference - for id := range b.refsMap { - if strings.HasPrefix(id.String(), shortID) { - if found != nil && found != id { + var found string + ids, err := b.listIDs() + if err != nil { + return nil, err + } + for _, id := range ids { + if strings.HasPrefix(id, shortID) { + if found != "" && found != id { return nil, fmt.Errorf("ambiguous reference found") } found = id } } - if found == nil { + if found == "" { return nil, unknownReference(shortID) } - return found, nil + ref := fromID(found) + return ref, nil } func (b *imageStore) referenceToID(ref reference.Reference) (ID, error) { if id, ok := ref.(ID); ok { return id, nil } - for id, refs := range b.refsMap { - for _, r := range refs { - if r == ref { - return id, nil - } - } - } - return ID{}, unknownReference(reference.FamiliarString(ref)) -} - -func (b *imageStore) storePaths(ref reference.Reference) ([]string, error) { - var paths []string - - id, err := b.referenceToID(ref) - if err != nil { - return nil, err - } - - if refs, exist := b.refsMap[id]; exist { - for _, rf := range refs { - path, err := b.storePath(rf) - if err != nil { - return nil, err - } - paths = append(paths, path) - } - } - - if len(paths) == 0 { - return nil, unknownReference(reference.FamiliarString(ref)) - } - return paths, nil -} - -func (b *imageStore) storePath(ref reference.Reference) (string, error) { - named, ok := ref.(reference.Named) - if !ok { - return filepath.Join(b.path, "_ids", ref.String()), nil - } - - name := strings.Replace(named.Name(), ":", "_", 1) - // A name is safe for use as a filesystem path (it is - // alphanumerics + "." + "/") except for the ":" used to - // separate domain from port which is not safe on Windows. - // Replace it with "_" which is not valid in the name. - // - // There can be at most 1 ":" in a valid reference so only - // replace one -- if there are more (and this wasn't caught - // when parsing the ref) then there will be errors when we try - // to use this as a path later. - storeDir := filepath.Join(b.path, filepath.FromSlash(name)) - - // We rely here on _ not being valid in a name meaning there can be no clashes due to nesting of repositories. - switch t := ref.(type) { - case reference.Digested: - digest := t.Digest() - storeDir = filepath.Join(storeDir, "_digests", digest.Algorithm().String(), digest.Encoded()) - case reference.Tagged: - storeDir = filepath.Join(storeDir, "_tags", t.Tag()) - default: - return "", errors.Errorf("%s: not tagged or digested", ref.String()) - } - - return storeDir, nil -} - -// scanAllBundles scans the bundle store directories and creates the internal map of App image -// references. This function must be called before any other public ImageStore interface method. -func (b *imageStore) scanAllBundles() error { - if err := filepath.Walk(b.path, b.processImageStoreFile); err != nil { - return err - } - return nil -} - -func (b *imageStore) processImageStoreFile(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - idRefPath := filepath.Join(b.path, "_ids") - - if info.IsDir() { - return nil - } - - if info.Name() == image.RelocationMapFilename { - return nil - } - - if !strings.HasSuffix(info.Name(), ".json") { - return nil - } - - if strings.HasPrefix(path, idRefPath) { - rel := path[len(idRefPath)+1:] - dg := strings.Split(filepath.ToSlash(rel), "/")[0] - id := ID{digest.NewDigestFromEncoded(digest.SHA256, dg)} - b.refsMap.appendRef(id, id) - return nil - } - - ref, err := b.pathToReference(path) - if err != nil { - return err - } - img, err := image.FromFile(path) - if err != nil { - return err - } - id, err := FromAppImage(img) - if err != nil { - return err - } - b.refsMap[id] = append(b.refsMap[id], ref) - - return nil + named := ref.(reference.Named) + digest, err := b.store.Get(reference.TagNameOnly(named)) + return ID(digest), err } -func (b *imageStore) pathToReference(path string) (reference.Named, error) { - // Clean the path and remove the local bundle store path - cleanpath := filepath.ToSlash(path) - cleanpath = strings.TrimPrefix(cleanpath, filepath.ToSlash(b.path)+"/") - - // get the hierarchy of directories, so we can get digest algorithm or tag - paths := strings.Split(cleanpath, "/") - if len(paths) < 3 { - return nil, fmt.Errorf("invalid path %q in the bundle store", path) - } - - // path must point to a json file - if !strings.Contains(paths[len(paths)-1], ".json") { - return nil, fmt.Errorf("invalid path %q, not referencing a CNAB bundle in json format", path) - } - - // remove the bundle.json filename - paths = paths[:len(paths)-1] - - name, err := reconstructNamedReference(path, paths) - if err != nil { - return nil, err - } - - return reference.ParseNamed(name) +func (b *imageStore) storePath(ref digest.Digest) string { + return filepath.Join(b.path, "contents", ref.Algorithm().String(), ref.Encoded()) } func (rm referencesMap) appendRef(id ID, ref reference.Reference) { @@ -389,27 +276,6 @@ func (rm referencesMap) removeRef(ref reference.Reference) { } } -func reconstructNamedReference(path string, paths []string) (string, error) { - name, paths := strings.Replace(paths[0], "_", ":", 1), paths[1:] - for i, p := range paths { - switch p { - case "_tags": - if i != len(paths)-2 { - return "", fmt.Errorf("invalid path %q in the bundle store", path) - } - return fmt.Sprintf("%s:%s", name, paths[i+1]), nil - case "_digests": - if i != len(paths)-3 { - return "", fmt.Errorf("invalid path %q in the bundle store", path) - } - return fmt.Sprintf("%s@%s:%s", name, paths[i+1], paths[i+2]), nil - default: - name += "/" + p - } - } - return name, nil -} - func containsRef(list []reference.Reference, ref reference.Reference) bool { for _, v := range list { if v == ref { diff --git a/internal/store/image_test.go b/internal/store/image_test.go index 9b76e8a6d..4b5fd63ab 100644 --- a/internal/store/image_test.go +++ b/internal/store/image_test.go @@ -2,10 +2,8 @@ package store import ( "fmt" - "io/ioutil" "os" "path" - "path/filepath" "testing" "github.com/docker/app/internal/image" @@ -33,28 +31,21 @@ func TestStoreAndReadBundle(t *testing.T) { testcases := []struct { name string ref reference.Named - path string }{ { name: "tagged", ref: parseRefOrDie(t, "my-repo/my-bundle:my-tag"), - path: dockerConfigDir.Join("app", "bundles", "docker.io", "my-repo", "my-bundle", "_tags", "my-tag", image.BundleFilename), }, { name: "digested", ref: parseRefOrDie(t, "my-repo/my-bundle@sha256:"+testSha), - path: dockerConfigDir.Join("app", "bundles", "docker.io", "my-repo", "my-bundle", "_digests", "sha256", testSha, image.BundleFilename), }, } for _, testcase := range testcases { t.Run(testcase.name, func(t *testing.T) { // Store the bundle - _, err = imageStore.Store(testcase.ref, expectedBundle) - assert.NilError(t, err) - - // Check the file exists - _, err = os.Stat(testcase.path) + _, err = imageStore.Store(expectedBundle, testcase.ref) assert.NilError(t, err) // Load it @@ -72,139 +63,6 @@ func parseRefOrDie(t *testing.T, ref string) reference.Named { return named } -func TestStorePath(t *testing.T) { - bs := &imageStore{path: "base-dir"} - for _, tc := range []struct { - Name string - Ref reference.Named - ExpectedSubpath string - ExpectedError string - }{ - // storePath expects a tagged or digested, i.e. the use of TagNameOnly to add :latest. Check that it rejects untagged refs - { - Name: "untagged", - Ref: parseRefOrDie(t, "foo"), - ExpectedError: "docker.io/library/foo: not tagged or digested", - }, - // Variants of a tagged ref - { - Name: "simple-tagged", - Ref: parseRefOrDie(t, "foo:latest"), - ExpectedSubpath: "docker.io/library/foo/_tags/latest", - }, - { - Name: "deep-simple-tagged", - Ref: parseRefOrDie(t, "user/foo/bar:latest"), - ExpectedSubpath: "docker.io/user/foo/bar/_tags/latest", - }, - { - Name: "host-and-tagged", - Ref: parseRefOrDie(t, "my.registry.example.com/foo:latest"), - ExpectedSubpath: "my.registry.example.com/foo/_tags/latest", - }, - { - Name: "host-port-and-tagged", - Ref: parseRefOrDie(t, "my.registry.example.com:5000/foo:latest"), - ExpectedSubpath: "my.registry.example.com_5000/foo/_tags/latest", - }, - // Variants of a digested ref - { - Name: "simple-digested", - Ref: parseRefOrDie(t, "foo@sha256:"+testSha), - ExpectedSubpath: "docker.io/library/foo/_digests/sha256/" + testSha, - }, - { - Name: "deep-simple-digested", - Ref: parseRefOrDie(t, "user/foo/bar@sha256:"+testSha), - ExpectedSubpath: "docker.io/user/foo/bar/_digests/sha256/" + testSha, - }, - { - Name: "host-and-digested", - Ref: parseRefOrDie(t, "my.registry.example.com/foo@sha256:"+testSha), - ExpectedSubpath: "my.registry.example.com/foo/_digests/sha256/" + testSha, - }, - { - Name: "host-port-and-digested", - Ref: parseRefOrDie(t, "my.registry.example.com:5000/foo@sha256:"+testSha), - ExpectedSubpath: "my.registry.example.com_5000/foo/_digests/sha256/" + testSha, - }, - // If both then digest takes precedence (tag is ignored) - { - Name: "simple-tagged-and-digested", - Ref: parseRefOrDie(t, "foo:latest@sha256:"+testSha), - ExpectedSubpath: "docker.io/library/foo/_digests/sha256/" + testSha, - }, - { - Name: "deep-simple-tagged-and-digested", - Ref: parseRefOrDie(t, "user/foo/bar:latest@sha256:"+testSha), - ExpectedSubpath: "docker.io/user/foo/bar/_digests/sha256/" + testSha, - }, - { - Name: "host-and-tagged-and-digested", - Ref: parseRefOrDie(t, "my.registry.example.com/foo:latest@sha256:"+testSha), - ExpectedSubpath: "my.registry.example.com/foo/_digests/sha256/" + testSha, - }, - { - Name: "host-port-and-tagged-and-digested", - Ref: parseRefOrDie(t, "my.registry.example.com:5000/foo:latest@sha256:"+testSha), - ExpectedSubpath: "my.registry.example.com_5000/foo/_digests/sha256/" + testSha, - }, - } { - t.Run(tc.Name, func(t *testing.T) { - path, err := bs.storePath(tc.Ref) - if tc.ExpectedError == "" { - assert.NilError(t, err) - assert.Equal(t, filepath.Join("base-dir", filepath.FromSlash(tc.ExpectedSubpath)), path) - } else { - assert.Error(t, err, tc.ExpectedError) - } - }) - } -} - -func TestPathToReference(t *testing.T) { - imageStore := &imageStore{path: "base-dir"} - - for _, tc := range []struct { - Name string - Path string - ExpectedError string - ExpectedName string - }{ - { - Name: "error on invalid path", - Path: "invalid", - ExpectedError: `invalid path "invalid" in the bundle store`, - }, { - Name: "error if file is not json", - Path: "registry/repo/name/_tags/file.xml", - ExpectedError: `invalid path "registry/repo/name/_tags/file.xml", not referencing a CNAB bundle in json format`, - }, { - Name: "return a reference from tagged", - Path: "docker.io/library/foo/_tags/latest/bundle.json", - ExpectedName: "docker.io/library/foo", - }, { - Name: "return a reference from digested", - Path: "docker.io/library/foo/_digests/sha256/" + testSha + "/bundle.json", - ExpectedName: "docker.io/library/foo", - }, - } { - t.Run(tc.Name, func(t *testing.T) { - ref, err := imageStore.pathToReference(tc.Path) - - if tc.ExpectedError != "" { - assert.Equal(t, err.Error(), tc.ExpectedError) - } else { - assert.NilError(t, err) - } - - if tc.ExpectedName != "" { - assert.Equal(t, ref.Name(), tc.ExpectedName) - } - }) - } -} - func TestList(t *testing.T) { dockerConfigDir := fs.NewDir(t, t.Name(), fs.WithMode(0755)) defer dockerConfigDir.Remove() @@ -226,7 +84,7 @@ func TestList(t *testing.T) { img := image.FromBundle(&bundle.Bundle{Name: "bundle-name"}) for _, ref := range refs { - _, err = imageStore.Store(ref, img) + _, err = imageStore.Store(img, ref) assert.NilError(t, err) } @@ -264,13 +122,13 @@ func TestRemove(t *testing.T) { img := image.FromBundle(&bundle.Bundle{Name: "bundle-name"}) for _, ref := range refs { - _, err = imageStore.Store(ref, img) + _, err = imageStore.Store(img, ref) assert.NilError(t, err) } t.Run("error on unknown", func(t *testing.T) { err := imageStore.Remove(parseRefOrDie(t, "my-repo/some-bundle:1.0.0"), false) - assert.Equal(t, err.Error(), "no such image my-repo/some-bundle:1.0.0") + assert.Equal(t, err.Error(), "reference does not exist") }) t.Run("remove tagged and digested", func(t *testing.T) { @@ -308,16 +166,18 @@ func TestRemoveById(t *testing.T) { assert.NilError(t, err) err = imageStore.Remove(idRef, false) - assert.Equal(t, err.Error(), fmt.Sprintf("no such image %q", reference.FamiliarString(idRef))) + assert.Equal(t, err.Error(), fmt.Sprintf("%s: reference not found", idRef.String())) }) t.Run("error on multiple repositories", func(t *testing.T) { img := image.FromBundle(&bundle.Bundle{Name: "bundle-name"}) idRef, err := FromAppImage(img) assert.NilError(t, err) - _, err = imageStore.Store(idRef, img) + _, err = imageStore.Store(img, idRef) + assert.NilError(t, err) + _, err = imageStore.Store(img, parseRefOrDie(t, "my-repo/a-bundle:my-tag")) assert.NilError(t, err) - _, err = imageStore.Store(parseRefOrDie(t, "my-repo/a-bundle:my-tag"), img) + _, err = imageStore.Store(img, parseRefOrDie(t, "my-repo/a-bundle:latest")) assert.NilError(t, err) err = imageStore.Remove(idRef, false) @@ -328,9 +188,9 @@ func TestRemoveById(t *testing.T) { img := image.FromBundle(&bundle.Bundle{Name: "bundle-name"}) idRef, err := FromAppImage(img) assert.NilError(t, err) - _, err = imageStore.Store(idRef, img) + _, err = imageStore.Store(img, idRef) assert.NilError(t, err) - _, err = imageStore.Store(parseRefOrDie(t, "my-repo/a-bundle:my-tag"), img) + _, err = imageStore.Store(img, parseRefOrDie(t, "my-repo/a-bundle:my-tag")) assert.NilError(t, err) err = imageStore.Remove(idRef, true) @@ -340,7 +200,7 @@ func TestRemoveById(t *testing.T) { t.Run("success when only one reference exists", func(t *testing.T) { img := image.FromBundle(&bundle.Bundle{Name: "other-bundle-name"}) ref := parseRefOrDie(t, "my-repo/other-bundle:my-tag") - _, err = imageStore.Store(ref, img) + _, err = imageStore.Store(img, ref) idRef, err := FromAppImage(img) assert.NilError(t, err) @@ -363,15 +223,15 @@ func TestLookUp(t *testing.T) { assert.NilError(t, err) img := image.FromBundle(&bundle.Bundle{Name: "bundle-name"}) // Adding the bundle referenced by id - id, err := imageStore.Store(nil, img) + id, err := imageStore.Store(img, nil) assert.NilError(t, err) // Adding the same bundle referenced by a tag ref := parseRefOrDie(t, "my-repo/a-bundle:my-tag") - _, err = imageStore.Store(ref, img) + _, err = imageStore.Store(img, ref) assert.NilError(t, err) // Adding the same bundle referenced by tag prefixed by docker.io/library dockerIoRef := parseRefOrDie(t, "docker.io/library/a-bundle:my-tag") - _, err = imageStore.Store(dockerIoRef, img) + _, err = imageStore.Store(img, dockerIoRef) assert.NilError(t, err) for _, tc := range []struct { @@ -419,7 +279,7 @@ func TestLookUp(t *testing.T) { { Name: "Unknown short ID", refOrID: "b4fcc3af", - ExpectedError: "b4fcc3af:latest: reference not found", + ExpectedError: "b4fcc3af: reference not found", ExpectedRef: nil, }, } { @@ -440,57 +300,6 @@ func TestLookUp(t *testing.T) { } } -func TestScanBundles(t *testing.T) { - dockerConfigDir := fs.NewDir(t, t.Name(), fs.WithMode(0755)) - defer dockerConfigDir.Remove() - - // Adding a bundle which should be referenced by id only - img1 := image.FromBundle(&bundle.Bundle{Name: "bundle-1"}) - id1, err := FromAppImage(img1) - assert.NilError(t, err) - dir1 := dockerConfigDir.Join("app", "bundles", "_ids", id1.String()) - assert.NilError(t, os.MkdirAll(dir1, 0755)) - assert.NilError(t, ioutil.WriteFile(filepath.Join(dir1, image.BundleFilename), []byte(`{"name": "bundle-1"}`), 0644)) - - // Adding a bundle which should be referenced by id and tag - img2 := image.FromBundle(&bundle.Bundle{Name: "bundle-2"}) - id2, err := FromAppImage(img2) - assert.NilError(t, err) - dir2 := dockerConfigDir.Join("app", "bundles", "_ids", id2.String()) - assert.NilError(t, os.MkdirAll(dir2, 0755)) - assert.NilError(t, ioutil.WriteFile(filepath.Join(dir2, image.BundleFilename), []byte(`{"name": "bundle-2"}`), 0644)) - dir2 = dockerConfigDir.Join("app", "bundles", "docker.io", "my-repo", "my-bundle", "_tags", "my-tag") - assert.NilError(t, os.MkdirAll(dir2, 0755)) - assert.NilError(t, ioutil.WriteFile(filepath.Join(dir2, image.BundleFilename), []byte(`{"name": "bundle-2"}`), 0644)) - - appstore, err := NewApplicationStore(dockerConfigDir.Path()) - assert.NilError(t, err) - imageStore, err := appstore.ImageStore() - assert.NilError(t, err) - - // Ensure List() and Read() function returns expected bundles - refs, err := imageStore.List() - assert.NilError(t, err) - expectedRefs := []string{id2.String(), "my-repo/my-bundle:my-tag", id1.String()} - refsAsString := func(references []reference.Reference) []string { - var rv []string - for _, r := range references { - rv = append(rv, reference.FamiliarString(r)) - } - return rv - } - assert.DeepEqual(t, refsAsString(refs), expectedRefs) - img, err := imageStore.Read(id1) - assert.NilError(t, err) - assert.DeepEqual(t, img, img1) - img, err = imageStore.Read(id2) - assert.NilError(t, err) - assert.DeepEqual(t, img, img2) - img, err = imageStore.Read(parseRefOrDie(t, "my-repo/my-bundle:my-tag")) - assert.NilError(t, err) - assert.DeepEqual(t, img, img2) -} - func TestAppendRemoveReference(t *testing.T) { id1, err := FromString("68720b2db729794a3521bc83e3699ac629f26beba6862b6ec491cd0d677d02a0") assert.NilError(t, err) diff --git a/vendor/github.com/docker/docker/reference/errors.go b/vendor/github.com/docker/docker/reference/errors.go new file mode 100644 index 000000000..2d294c672 --- /dev/null +++ b/vendor/github.com/docker/docker/reference/errors.go @@ -0,0 +1,25 @@ +package reference // import "github.com/docker/docker/reference" + +type notFoundError string + +func (e notFoundError) Error() string { + return string(e) +} + +func (notFoundError) NotFound() {} + +type invalidTagError string + +func (e invalidTagError) Error() string { + return string(e) +} + +func (invalidTagError) InvalidParameter() {} + +type conflictingTagError string + +func (e conflictingTagError) Error() string { + return string(e) +} + +func (conflictingTagError) Conflict() {} diff --git a/vendor/github.com/docker/docker/reference/store.go b/vendor/github.com/docker/docker/reference/store.go new file mode 100644 index 000000000..d6ef6697f --- /dev/null +++ b/vendor/github.com/docker/docker/reference/store.go @@ -0,0 +1,348 @@ +package reference // import "github.com/docker/docker/reference" + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "sync" + + "github.com/docker/distribution/reference" + "github.com/docker/docker/pkg/ioutils" + digest "github.com/opencontainers/go-digest" + "github.com/pkg/errors" +) + +var ( + // ErrDoesNotExist is returned if a reference is not found in the + // store. + ErrDoesNotExist notFoundError = "reference does not exist" +) + +// An Association is a tuple associating a reference with an image ID. +type Association struct { + Ref reference.Named + ID digest.Digest +} + +// Store provides the set of methods which can operate on a reference store. +type Store interface { + References(id digest.Digest) []reference.Named + ReferencesByName(ref reference.Named) []Association + AddTag(ref reference.Named, id digest.Digest, force bool) error + AddDigest(ref reference.Canonical, id digest.Digest, force bool) error + Delete(ref reference.Named) (bool, error) + Get(ref reference.Named) (digest.Digest, error) +} + +type store struct { + mu sync.RWMutex + // jsonPath is the path to the file where the serialized tag data is + // stored. + jsonPath string + // Repositories is a map of repositories, indexed by name. + Repositories map[string]repository + // referencesByIDCache is a cache of references indexed by ID, to speed + // up References. + referencesByIDCache map[digest.Digest]map[string]reference.Named +} + +// Repository maps tags to digests. The key is a stringified Reference, +// including the repository name. +type repository map[string]digest.Digest + +type lexicalRefs []reference.Named + +func (a lexicalRefs) Len() int { return len(a) } +func (a lexicalRefs) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a lexicalRefs) Less(i, j int) bool { + return a[i].String() < a[j].String() +} + +type lexicalAssociations []Association + +func (a lexicalAssociations) Len() int { return len(a) } +func (a lexicalAssociations) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a lexicalAssociations) Less(i, j int) bool { + return a[i].Ref.String() < a[j].Ref.String() +} + +// NewReferenceStore creates a new reference store, tied to a file path where +// the set of references are serialized in JSON format. +func NewReferenceStore(jsonPath string) (Store, error) { + abspath, err := filepath.Abs(jsonPath) + if err != nil { + return nil, err + } + + store := &store{ + jsonPath: abspath, + Repositories: make(map[string]repository), + referencesByIDCache: make(map[digest.Digest]map[string]reference.Named), + } + // Load the json file if it exists, otherwise create it. + if err := store.reload(); os.IsNotExist(err) { + if err := store.save(); err != nil { + return nil, err + } + } else if err != nil { + return nil, err + } + return store, nil +} + +// AddTag adds a tag reference to the store. If force is set to true, existing +// references can be overwritten. This only works for tags, not digests. +func (store *store) AddTag(ref reference.Named, id digest.Digest, force bool) error { + if _, isCanonical := ref.(reference.Canonical); isCanonical { + return errors.WithStack(invalidTagError("refusing to create a tag with a digest reference")) + } + return store.addReference(reference.TagNameOnly(ref), id, force) +} + +// AddDigest adds a digest reference to the store. +func (store *store) AddDigest(ref reference.Canonical, id digest.Digest, force bool) error { + return store.addReference(ref, id, force) +} + +func favorDigest(originalRef reference.Named) (reference.Named, error) { + ref := originalRef + // If the reference includes a digest and a tag, we must store only the + // digest. + canonical, isCanonical := originalRef.(reference.Canonical) + _, isNamedTagged := originalRef.(reference.NamedTagged) + + if isCanonical && isNamedTagged { + trimmed, err := reference.WithDigest(reference.TrimNamed(canonical), canonical.Digest()) + if err != nil { + // should never happen + return originalRef, err + } + ref = trimmed + } + return ref, nil +} + +func (store *store) addReference(ref reference.Named, id digest.Digest, force bool) error { + ref, err := favorDigest(ref) + if err != nil { + return err + } + + refName := reference.FamiliarName(ref) + refStr := reference.FamiliarString(ref) + + if refName == string(digest.Canonical) { + return errors.WithStack(invalidTagError("refusing to create an ambiguous tag using digest algorithm as name")) + } + + store.mu.Lock() + defer store.mu.Unlock() + + repository, exists := store.Repositories[refName] + if !exists || repository == nil { + repository = make(map[string]digest.Digest) + store.Repositories[refName] = repository + } + + oldID, exists := repository[refStr] + + if exists { + if oldID == id { + // Nothing to do. The caller may have checked for this using store.Get in advance, but store.mu was unlocked in the meantime, so this can legitimately happen nevertheless. + return nil + } + + // force only works for tags + if digested, isDigest := ref.(reference.Canonical); isDigest { + return errors.WithStack(conflictingTagError("Cannot overwrite digest " + digested.Digest().String())) + } + + if !force { + return errors.WithStack( + conflictingTagError( + fmt.Sprintf("Conflict: Tag %s is already set to image %s, if you want to replace it, please use the force option", refStr, oldID.String()), + ), + ) + } + + if store.referencesByIDCache[oldID] != nil { + delete(store.referencesByIDCache[oldID], refStr) + if len(store.referencesByIDCache[oldID]) == 0 { + delete(store.referencesByIDCache, oldID) + } + } + } + + repository[refStr] = id + if store.referencesByIDCache[id] == nil { + store.referencesByIDCache[id] = make(map[string]reference.Named) + } + store.referencesByIDCache[id][refStr] = ref + + return store.save() +} + +// Delete deletes a reference from the store. It returns true if a deletion +// happened, or false otherwise. +func (store *store) Delete(ref reference.Named) (bool, error) { + ref, err := favorDigest(ref) + if err != nil { + return false, err + } + + ref = reference.TagNameOnly(ref) + + refName := reference.FamiliarName(ref) + refStr := reference.FamiliarString(ref) + + store.mu.Lock() + defer store.mu.Unlock() + + repository, exists := store.Repositories[refName] + if !exists { + return false, ErrDoesNotExist + } + + if id, exists := repository[refStr]; exists { + delete(repository, refStr) + if len(repository) == 0 { + delete(store.Repositories, refName) + } + if store.referencesByIDCache[id] != nil { + delete(store.referencesByIDCache[id], refStr) + if len(store.referencesByIDCache[id]) == 0 { + delete(store.referencesByIDCache, id) + } + } + return true, store.save() + } + + return false, ErrDoesNotExist +} + +// Get retrieves an item from the store by reference +func (store *store) Get(ref reference.Named) (digest.Digest, error) { + if canonical, ok := ref.(reference.Canonical); ok { + // If reference contains both tag and digest, only + // lookup by digest as it takes precedence over + // tag, until tag/digest combos are stored. + if _, ok := ref.(reference.Tagged); ok { + var err error + ref, err = reference.WithDigest(reference.TrimNamed(canonical), canonical.Digest()) + if err != nil { + return "", err + } + } + } else { + ref = reference.TagNameOnly(ref) + } + + refName := reference.FamiliarName(ref) + refStr := reference.FamiliarString(ref) + + store.mu.RLock() + defer store.mu.RUnlock() + + repository, exists := store.Repositories[refName] + if !exists || repository == nil { + return "", ErrDoesNotExist + } + + id, exists := repository[refStr] + if !exists { + return "", ErrDoesNotExist + } + + return id, nil +} + +// References returns a slice of references to the given ID. The slice +// will be nil if there are no references to this ID. +func (store *store) References(id digest.Digest) []reference.Named { + store.mu.RLock() + defer store.mu.RUnlock() + + // Convert the internal map to an array for two reasons: + // 1) We must not return a mutable + // 2) It would be ugly to expose the extraneous map keys to callers. + + var references []reference.Named + for _, ref := range store.referencesByIDCache[id] { + references = append(references, ref) + } + + sort.Sort(lexicalRefs(references)) + + return references +} + +// ReferencesByName returns the references for a given repository name. +// If there are no references known for this repository name, +// ReferencesByName returns nil. +func (store *store) ReferencesByName(ref reference.Named) []Association { + refName := reference.FamiliarName(ref) + + store.mu.RLock() + defer store.mu.RUnlock() + + repository, exists := store.Repositories[refName] + if !exists { + return nil + } + + var associations []Association + for refStr, refID := range repository { + ref, err := reference.ParseNormalizedNamed(refStr) + if err != nil { + // Should never happen + return nil + } + associations = append(associations, + Association{ + Ref: ref, + ID: refID, + }) + } + + sort.Sort(lexicalAssociations(associations)) + + return associations +} + +func (store *store) save() error { + // Store the json + jsonData, err := json.Marshal(store) + if err != nil { + return err + } + return ioutils.AtomicWriteFile(store.jsonPath, jsonData, 0600) +} + +func (store *store) reload() error { + f, err := os.Open(store.jsonPath) + if err != nil { + return err + } + defer f.Close() + if err := json.NewDecoder(f).Decode(&store); err != nil { + return err + } + + for _, repository := range store.Repositories { + for refStr, refID := range repository { + ref, err := reference.ParseNormalizedNamed(refStr) + if err != nil { + // Should never happen + continue + } + if store.referencesByIDCache[refID] == nil { + store.referencesByIDCache[refID] = make(map[string]reference.Named) + } + store.referencesByIDCache[refID][refStr] = ref + } + } + + return nil +}