From 523b8b44a242a36afdb6a3fd0d3aba07ea4a3d87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miloslav=20Trma=C4=8D?= Date: Fri, 14 Jun 2019 17:24:20 +0200 Subject: [PATCH] Update buildah to 1.8.4, c/storage to 1.12.10 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Miloslav Trmač --- vendor.conf | 4 +- .../containers/buildah/pkg/unshare/unshare.c | 13 +- .../containers/buildah/pkg/unshare/unshare.go | 45 ++++-- .../github.com/containers/buildah/vendor.conf | 7 +- .../containers/storage/containers.go | 4 + .../containers/storage/drivers/aufs/aufs.go | 3 + .../containers/storage/drivers/chown.go | 3 + .../containers/storage/drivers/driver.go | 1 + .../storage/drivers/overlay/overlay.go | 77 ++++++++++- .../containers/storage/drivers/vfs/driver.go | 7 +- .../github.com/containers/storage/images.go | 9 ++ .../github.com/containers/storage/layers.go | 14 +- .../github.com/containers/storage/lockfile.go | 4 + .../containers/storage/lockfile_linux.go | 20 --- .../containers/storage/lockfile_otherunix.go | 19 --- .../containers/storage/lockfile_unix.go | 41 +++++- .../containers/storage/lockfile_windows.go | 6 + .../storage/pkg/chrootarchive/archive.go | 36 ++++- .../storage/pkg/chrootarchive/archive_unix.go | 130 +++++++++++++++++- .../pkg/chrootarchive/archive_windows.go | 9 +- .../storage/pkg/chrootarchive/chroot_unix.go | 6 +- .../storage/pkg/chrootarchive/init_unix.go | 1 + .../storage/pkg/system/stat_unix.go | 12 ++ vendor/github.com/containers/storage/store.go | 34 +++-- vendor/github.com/containers/storage/utils.go | 16 ++- 25 files changed, 420 insertions(+), 101 deletions(-) delete mode 100644 vendor/github.com/containers/storage/lockfile_linux.go delete mode 100644 vendor/github.com/containers/storage/lockfile_otherunix.go diff --git a/vendor.conf b/vendor.conf index ffdfc82891..0fdd146c00 100644 --- a/vendor.conf +++ b/vendor.conf @@ -3,13 +3,13 @@ github.com/urfave/cli v1.20.0 github.com/kr/pretty v0.1.0 github.com/kr/text v0.1.0 github.com/containers/image v2.0.0 -github.com/containers/buildah 810efa340ab43753034e2ed08ec290e4abab7e72 +github.com/containers/buildah v1.8.4 github.com/vbauerster/mpb v3.3.4 github.com/mattn/go-isatty v0.0.4 github.com/VividCortex/ewma v1.1.1 golang.org/x/sync 42b317875d0fa942474b76e1b46a6060d720ae6e github.com/opencontainers/go-digest c9281466c8b2f606084ac71339773efd177436e7 -github.com/containers/storage v1.12.3 +github.com/containers/storage v1.12.10 github.com/sirupsen/logrus v1.0.0 github.com/go-check/check v1 github.com/stretchr/testify v1.1.3 diff --git a/vendor/github.com/containers/buildah/pkg/unshare/unshare.c b/vendor/github.com/containers/buildah/pkg/unshare/unshare.c index 67a3e0e4d0..fd0d48d435 100644 --- a/vendor/github.com/containers/buildah/pkg/unshare/unshare.c +++ b/vendor/github.com/containers/buildah/pkg/unshare/unshare.c @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include #include #include @@ -14,6 +14,17 @@ #include #include +/* Open Source projects like conda-forge, want to package podman and are based + off of centos:6, Conda-force has minimal libc requirements and is lacking + the memfd.h file, so we use mmam.h +*/ +#ifndef MFD_ALLOW_SEALING +#define MFD_ALLOW_SEALING 2U +#endif +#ifndef MFD_CLOEXEC +#define MFD_CLOEXEC 1U +#endif + #ifndef F_LINUX_SPECIFIC_BASE #define F_LINUX_SPECIFIC_BASE 1024 #endif diff --git a/vendor/github.com/containers/buildah/pkg/unshare/unshare.go b/vendor/github.com/containers/buildah/pkg/unshare/unshare.go index 33232740e2..21b102cf56 100644 --- a/vendor/github.com/containers/buildah/pkg/unshare/unshare.go +++ b/vendor/github.com/containers/buildah/pkg/unshare/unshare.go @@ -64,6 +64,7 @@ func (c *Cmd) Start() error { if os.Geteuid() != 0 { c.Env = append(c.Env, "_CONTAINERS_USERNS_CONFIGURED=done") c.Env = append(c.Env, fmt.Sprintf("_CONTAINERS_ROOTLESS_UID=%d", os.Geteuid())) + c.Env = append(c.Env, fmt.Sprintf("_CONTAINERS_ROOTLESS_GID=%d", os.Getegid())) } // Create the pipe for reading the child's PID. @@ -183,6 +184,7 @@ func (c *Cmd) Start() error { for _, m := range c.GidMappings { fmt.Fprintf(g, "%d %d %d\n", m.ContainerID, m.HostID, m.Size) } + gidmapSet := false // Set the GID map. if c.UseNewgidmap { cmd := exec.Command("newgidmap", append([]string{pidString}, strings.Fields(strings.Replace(g.String(), "\n", " ", -1))...)...) @@ -190,11 +192,28 @@ func (c *Cmd) Start() error { cmd.Stdout = g cmd.Stderr = g err := cmd.Run() - if err != nil { - fmt.Fprintf(continueWrite, "error running newgidmap: %v: %s", err, g.String()) - return errors.Wrapf(err, "error running newgidmap: %s", g.String()) + if err == nil { + gidmapSet = true + } else { + logrus.Warnf("error running newgidmap: %v: %s", err, g.String()) + logrus.Warnf("falling back to single mapping") + g.Reset() + g.Write([]byte(fmt.Sprintf("0 %d 1\n", os.Getegid()))) + } + } + if !gidmapSet { + if c.UseNewgidmap { + setgroups, err := os.OpenFile(fmt.Sprintf("/proc/%s/setgroups", pidString), os.O_TRUNC|os.O_WRONLY, 0) + if err != nil { + fmt.Fprintf(continueWrite, "error opening /proc/%s/setgroups: %v", pidString, err) + return errors.Wrapf(err, "error opening /proc/%s/setgroups", pidString) + } + defer setgroups.Close() + if _, err := fmt.Fprintf(setgroups, "deny"); err != nil { + fmt.Fprintf(continueWrite, "error writing 'deny' to /proc/%s/setgroups: %v", pidString, err) + return errors.Wrapf(err, "error writing 'deny' to /proc/%s/setgroups", pidString) + } } - } else { gidmap, err := os.OpenFile(fmt.Sprintf("/proc/%s/gid_map", pidString), os.O_TRUNC|os.O_WRONLY, 0) if err != nil { fmt.Fprintf(continueWrite, "error opening /proc/%s/gid_map: %v", pidString, err) @@ -214,6 +233,7 @@ func (c *Cmd) Start() error { for _, m := range c.UidMappings { fmt.Fprintf(u, "%d %d %d\n", m.ContainerID, m.HostID, m.Size) } + uidmapSet := false // Set the GID map. if c.UseNewuidmap { cmd := exec.Command("newuidmap", append([]string{pidString}, strings.Fields(strings.Replace(u.String(), "\n", " ", -1))...)...) @@ -221,11 +241,16 @@ func (c *Cmd) Start() error { cmd.Stdout = u cmd.Stderr = u err := cmd.Run() - if err != nil { - fmt.Fprintf(continueWrite, "error running newuidmap: %v: %s", err, u.String()) - return errors.Wrapf(err, "error running newuidmap: %s", u.String()) + if err == nil { + uidmapSet = true + } else { + logrus.Warnf("error running newuidmap: %v: %s", err, u.String()) + logrus.Warnf("falling back to single mapping") + u.Reset() + u.Write([]byte(fmt.Sprintf("0 %d 1\n", os.Geteuid()))) } - } else { + } + if !uidmapSet { uidmap, err := os.OpenFile(fmt.Sprintf("/proc/%s/uid_map", pidString), os.O_TRUNC|os.O_WRONLY, 0) if err != nil { fmt.Fprintf(continueWrite, "error opening /proc/%s/uid_map: %v", pidString, err) @@ -354,7 +379,9 @@ func MaybeReexecUsingUserNamespace(evenForRoot bool) { // range in /etc/subuid and /etc/subgid file is a starting host // ID and a range size. uidmap, gidmap, err = GetSubIDMappings(me.Username, me.Username) - bailOnError(err, "error reading allowed ID mappings") + if err != nil { + logrus.Warnf("error reading allowed ID mappings: %v", err) + } if len(uidmap) == 0 { logrus.Warnf("Found no UID ranges set aside for user %q in /etc/subuid.", me.Username) } diff --git a/vendor/github.com/containers/buildah/vendor.conf b/vendor/github.com/containers/buildah/vendor.conf index a77130acb5..88148947a7 100644 --- a/vendor/github.com/containers/buildah/vendor.conf +++ b/vendor/github.com/containers/buildah/vendor.conf @@ -3,12 +3,12 @@ github.com/blang/semver v3.5.0 github.com/BurntSushi/toml v0.2.0 github.com/containerd/continuity 004b46473808b3e7a4a3049c20e4376c91eb966d github.com/containernetworking/cni v0.7.0-rc2 -github.com/containers/image f52cf78ebfa1916da406f8b6210d8f7764ec1185 +github.com/containers/image v2.0.0 +github.com/cyphar/filepath-securejoin v0.2.1 github.com/vbauerster/mpb v3.3.4 github.com/mattn/go-isatty v0.0.4 github.com/VividCortex/ewma v1.1.1 -github.com/boltdb/bolt v1.3.1 -github.com/containers/storage v1.12.2 +github.com/containers/storage v1.12.10 github.com/docker/distribution 5f6282db7d65e6d72ad7c2cc66310724a57be716 github.com/docker/docker 54dddadc7d5d89fe0be88f76979f6f6ab0dede83 github.com/docker/docker-credential-helpers v0.6.1 @@ -66,3 +66,4 @@ github.com/onsi/gomega v1.4.3 github.com/spf13/cobra v0.0.3 github.com/spf13/pflag v1.0.3 github.com/ishidawataru/sctp 07191f837fedd2f13d1ec7b5f885f0f3ec54b1cb +github.com/etcd-io/bbolt v1.3.2 diff --git a/vendor/github.com/containers/storage/containers.go b/vendor/github.com/containers/storage/containers.go index bbac78b60b..e695523613 100644 --- a/vendor/github.com/containers/storage/containers.go +++ b/vendor/github.com/containers/storage/containers.go @@ -572,6 +572,10 @@ func (r *containerStore) Lock() { r.lockfile.Lock() } +func (r *containerStore) RecursiveLock() { + r.lockfile.RecursiveLock() +} + func (r *containerStore) RLock() { r.lockfile.RLock() } diff --git a/vendor/github.com/containers/storage/drivers/aufs/aufs.go b/vendor/github.com/containers/storage/drivers/aufs/aufs.go index e821bc0c5d..353d1707aa 100644 --- a/vendor/github.com/containers/storage/drivers/aufs/aufs.go +++ b/vendor/github.com/containers/storage/drivers/aufs/aufs.go @@ -255,6 +255,9 @@ func (a *Driver) AdditionalImageStores() []string { // CreateFromTemplate creates a layer with the same contents and parent as another layer. func (a *Driver) CreateFromTemplate(id, template string, templateIDMappings *idtools.IDMappings, parent string, parentIDMappings *idtools.IDMappings, opts *graphdriver.CreateOpts, readWrite bool) error { + if opts == nil { + opts = &graphdriver.CreateOpts{} + } return graphdriver.NaiveCreateFromTemplate(a, id, template, templateIDMappings, parent, parentIDMappings, opts, readWrite) } diff --git a/vendor/github.com/containers/storage/drivers/chown.go b/vendor/github.com/containers/storage/drivers/chown.go index 4d4011ee0c..f2f1ec3868 100644 --- a/vendor/github.com/containers/storage/drivers/chown.go +++ b/vendor/github.com/containers/storage/drivers/chown.go @@ -55,6 +55,9 @@ func chownByMapsMain() { if err != nil { return fmt.Errorf("error walking to %q: %v", path, err) } + if path == "." { + return nil + } return platformLChown(path, info, toHost, toContainer) } if err := filepath.Walk(".", chown); err != nil { diff --git a/vendor/github.com/containers/storage/drivers/driver.go b/vendor/github.com/containers/storage/drivers/driver.go index dda1725747..e8f8bd5a76 100644 --- a/vendor/github.com/containers/storage/drivers/driver.go +++ b/vendor/github.com/containers/storage/drivers/driver.go @@ -40,6 +40,7 @@ var ( type CreateOpts struct { MountLabel string StorageOpt map[string]string + *idtools.IDMappings } // MountOpts contains optional arguments for LayerStope.Mount() methods. diff --git a/vendor/github.com/containers/storage/drivers/overlay/overlay.go b/vendor/github.com/containers/storage/drivers/overlay/overlay.go index 69036a5c13..ef83b6c876 100644 --- a/vendor/github.com/containers/storage/drivers/overlay/overlay.go +++ b/vendor/github.com/containers/storage/drivers/overlay/overlay.go @@ -16,7 +16,7 @@ import ( "sync" "syscall" - "github.com/containers/storage/drivers" + graphdriver "github.com/containers/storage/drivers" "github.com/containers/storage/drivers/overlayutils" "github.com/containers/storage/drivers/quota" "github.com/containers/storage/pkg/archive" @@ -320,6 +320,8 @@ func supportsOverlay(home string, homeMagic graphdriver.FsMagic, rootUID, rootGI mergedDir := filepath.Join(layerDir, "merged") lower1Dir := filepath.Join(layerDir, "lower1") lower2Dir := filepath.Join(layerDir, "lower2") + upperDir := filepath.Join(layerDir, "upper") + workDir := filepath.Join(layerDir, "work") defer func() { // Permitted to fail, since the various subdirectories // can be empty or not even there, and the home might @@ -331,7 +333,9 @@ func supportsOverlay(home string, homeMagic graphdriver.FsMagic, rootUID, rootGI _ = idtools.MkdirAs(mergedDir, 0700, rootUID, rootGID) _ = idtools.MkdirAs(lower1Dir, 0700, rootUID, rootGID) _ = idtools.MkdirAs(lower2Dir, 0700, rootUID, rootGID) - flags := fmt.Sprintf("lowerdir=%s:%s", lower1Dir, lower2Dir) + _ = idtools.MkdirAs(upperDir, 0700, rootUID, rootGID) + _ = idtools.MkdirAs(workDir, 0700, rootUID, rootGID) + flags := fmt.Sprintf("lowerdir=%s:%s,upperdir=%s,workdir=%s", lower1Dir, lower2Dir, upperDir, workDir) if len(flags) < unix.Getpagesize() { err := mountFrom(filepath.Dir(home), "overlay", mergedDir, "overlay", 0, flags) if err == nil { @@ -341,7 +345,7 @@ func supportsOverlay(home string, homeMagic graphdriver.FsMagic, rootUID, rootGI logrus.Debugf("overlay test mount with multiple lowers failed %v", err) } } - flags = fmt.Sprintf("lowerdir=%s", lower1Dir) + flags = fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", lower1Dir, upperDir, workDir) if len(flags) < unix.Getpagesize() { err := mountFrom(filepath.Dir(home), "overlay", mergedDir, "overlay", 0, flags) if err == nil { @@ -470,10 +474,22 @@ func (d *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) (retErr func (d *Driver) create(id, parent string, opts *graphdriver.CreateOpts) (retErr error) { dir := d.dir(id) - rootUID, rootGID, err := idtools.GetRootUIDGID(d.uidMaps, d.gidMaps) + uidMaps := d.uidMaps + gidMaps := d.gidMaps + + if opts != nil && opts.IDMappings != nil { + uidMaps = opts.IDMappings.UIDs() + gidMaps = opts.IDMappings.GIDs() + } + + rootUID, rootGID, err := idtools.GetRootUIDGID(uidMaps, gidMaps) if err != nil { return err } + // Make the link directory if it does not exist + if err := idtools.MkdirAllAs(path.Join(d.home, linkDir), 0700, rootUID, rootGID); err != nil && !os.IsExist(err) { + return err + } if err := idtools.MkdirAllAs(path.Dir(dir), 0700, rootUID, rootGID); err != nil { return err } @@ -677,6 +693,48 @@ func (d *Driver) Remove(id string) error { return nil } +// recreateSymlinks goes through the driver's home directory and checks if the diff directory +// under each layer has a symlink created for it under the linkDir. If the symlink does not +// exist, it creates them +func (d *Driver) recreateSymlinks() error { + // List all the directories under the home directory + dirs, err := ioutil.ReadDir(d.home) + if err != nil { + return fmt.Errorf("error reading driver home directory %q: %v", d.home, err) + } + // This makes the link directory if it doesn't exist + rootUID, rootGID, err := idtools.GetRootUIDGID(d.uidMaps, d.gidMaps) + if err != nil { + return err + } + if err := idtools.MkdirAllAs(path.Join(d.home, linkDir), 0700, rootUID, rootGID); err != nil && !os.IsExist(err) { + return err + } + for _, dir := range dirs { + // Skip over the linkDir and anything that is not a directory + if dir.Name() == linkDir || !dir.Mode().IsDir() { + continue + } + // Read the "link" file under each layer to get the name of the symlink + data, err := ioutil.ReadFile(path.Join(d.dir(dir.Name()), "link")) + if err != nil { + return fmt.Errorf("error reading name of symlink for %q: %v", dir, err) + } + linkPath := path.Join(d.home, linkDir, strings.Trim(string(data), "\n")) + // Check if the symlink exists, and if it doesn't create it again with the name we + // got from the "link" file + _, err = os.Stat(linkPath) + if err != nil && os.IsNotExist(err) { + if err := os.Symlink(path.Join("..", dir.Name(), "diff"), linkPath); err != nil { + return err + } + } else if err != nil { + return fmt.Errorf("error trying to stat %q: %v", linkPath, err) + } + } + return nil +} + // Get creates and mounts the required file system for the given id and returns the mount path. func (d *Driver) Get(id string, options graphdriver.MountOpts) (_ string, retErr error) { return d.get(id, false, options) @@ -732,7 +790,16 @@ func (d *Driver) get(id string, disableShifting bool, options graphdriver.MountO } lower = "" } - if lower == "" { + // if it is a "not found" error, that means the symlinks were lost in a sudden reboot + // so call the recreateSymlinks function to go through all the layer dirs and recreate + // the symlinks with the name from their respective "link" files + if lower == "" && os.IsNotExist(err) { + logrus.Warnf("Can't stat lower layer %q because it does not exist. Going through storage to recreate the missing symlinks.", newpath) + if err := d.recreateSymlinks(); err != nil { + return "", fmt.Errorf("error recreating the missing symlinks: %v", err) + } + lower = newpath + } else if lower == "" { return "", fmt.Errorf("Can't stat lower layer %q: %v", newpath, err) } } else { diff --git a/vendor/github.com/containers/storage/drivers/vfs/driver.go b/vendor/github.com/containers/storage/drivers/vfs/driver.go index 5941ccc170..9e256858c1 100644 --- a/vendor/github.com/containers/storage/drivers/vfs/driver.go +++ b/vendor/github.com/containers/storage/drivers/vfs/driver.go @@ -123,8 +123,13 @@ func (d *Driver) create(id, parent string, opts *graphdriver.CreateOpts, ro bool return fmt.Errorf("--storage-opt is not supported for vfs") } + idMappings := d.idMappings + if opts != nil && opts.IDMappings != nil { + idMappings = opts.IDMappings + } + dir := d.dir(id) - rootIDs := d.idMappings.RootPair() + rootIDs := idMappings.RootPair() if err := idtools.MkdirAllAndChown(filepath.Dir(dir), 0700, rootIDs); err != nil { return err } diff --git a/vendor/github.com/containers/storage/images.go b/vendor/github.com/containers/storage/images.go index 38b5a3ef3b..6f487504a5 100644 --- a/vendor/github.com/containers/storage/images.go +++ b/vendor/github.com/containers/storage/images.go @@ -82,6 +82,9 @@ type Image struct { // is set before using it. Created time.Time `json:"created,omitempty"` + // ReadOnly is true if this image resides in a read-only layer store. + ReadOnly bool `json:"-"` + Flags map[string]interface{} `json:"flags,omitempty"` } @@ -159,6 +162,7 @@ func copyImage(i *Image) *Image { BigDataSizes: copyStringInt64Map(i.BigDataSizes), BigDataDigests: copyStringDigestMap(i.BigDataDigests), Created: i.Created, + ReadOnly: i.ReadOnly, Flags: copyStringInterfaceMap(i.Flags), } } @@ -269,6 +273,7 @@ func (r *imageStore) Load() error { list := digests[digest] digests[digest] = append(list, image) } + image.ReadOnly = !r.IsReadWrite() } } if shouldSave && (!r.IsReadWrite() || !r.Locked()) { @@ -739,6 +744,10 @@ func (r *imageStore) Lock() { r.lockfile.Lock() } +func (r *imageStore) RecursiveLock() { + r.lockfile.RecursiveLock() +} + func (r *imageStore) RLock() { r.lockfile.RLock() } diff --git a/vendor/github.com/containers/storage/layers.go b/vendor/github.com/containers/storage/layers.go index 110e737b2e..fb79238cd0 100644 --- a/vendor/github.com/containers/storage/layers.go +++ b/vendor/github.com/containers/storage/layers.go @@ -103,6 +103,9 @@ type Layer struct { // for use inside of a user namespace where UID mapping is being used. UIDMap []idtools.IDMap `json:"uidmap,omitempty"` GIDMap []idtools.IDMap `json:"gidmap,omitempty"` + + // ReadOnly is true if this layer resides in a read-only layer store. + ReadOnly bool `json:"-"` } type layerMountPoint struct { @@ -259,6 +262,7 @@ func copyLayer(l *Layer) *Layer { UncompressedDigest: l.UncompressedDigest, UncompressedSize: l.UncompressedSize, CompressionType: l.CompressionType, + ReadOnly: l.ReadOnly, Flags: copyStringInterfaceMap(l.Flags), UIDMap: copyIDMap(l.UIDMap), GIDMap: copyIDMap(l.GIDMap), @@ -318,6 +322,7 @@ func (r *layerStore) Load() error { if layer.MountLabel != "" { label.ReserveLabel(layer.MountLabel) } + layer.ReadOnly = !r.IsReadWrite() } err = nil } @@ -402,12 +407,10 @@ func (r *layerStore) Save() error { if err != nil { return err } + defer r.Touch() if err := ioutils.AtomicWriteFile(rpath, jldata, 0600); err != nil { return err } - if !r.IsReadWrite() { - return nil - } r.mountsLockfile.Lock() defer r.mountsLockfile.Unlock() defer r.mountsLockfile.Touch() @@ -614,6 +617,7 @@ func (r *layerStore) Put(id string, parentLayer *Layer, names []string, mountLab opts := drivers.CreateOpts{ MountLabel: mountLabel, StorageOpt: options, + IDMappings: idMappings, } if moreOptions.TemplateLayer != "" { if err = r.driver.CreateFromTemplate(id, moreOptions.TemplateLayer, templateIDMappings, parent, parentMappings, &opts, writeable); err != nil { @@ -1305,6 +1309,10 @@ func (r *layerStore) Lock() { r.lockfile.Lock() } +func (r *layerStore) RecursiveLock() { + r.lockfile.RecursiveLock() +} + func (r *layerStore) RLock() { r.lockfile.RLock() } diff --git a/vendor/github.com/containers/storage/lockfile.go b/vendor/github.com/containers/storage/lockfile.go index ed8753337f..c4f1b5549d 100644 --- a/vendor/github.com/containers/storage/lockfile.go +++ b/vendor/github.com/containers/storage/lockfile.go @@ -15,6 +15,10 @@ type Locker interface { // Acquire a writer lock. Lock() + // Acquire a writer lock recursively, allowing for recursive acquisitions + // within the same process space. + RecursiveLock() + // Unlock the lock. Unlock() diff --git a/vendor/github.com/containers/storage/lockfile_linux.go b/vendor/github.com/containers/storage/lockfile_linux.go deleted file mode 100644 index 903387c661..0000000000 --- a/vendor/github.com/containers/storage/lockfile_linux.go +++ /dev/null @@ -1,20 +0,0 @@ -// +build linux solaris - -package storage - -import ( - "time" - - "golang.org/x/sys/unix" -) - -// TouchedSince indicates if the lock file has been touched since the specified time -func (l *lockfile) TouchedSince(when time.Time) bool { - st := unix.Stat_t{} - err := unix.Fstat(int(l.fd), &st) - if err != nil { - return true - } - touched := time.Unix(st.Mtim.Unix()) - return when.Before(touched) -} diff --git a/vendor/github.com/containers/storage/lockfile_otherunix.go b/vendor/github.com/containers/storage/lockfile_otherunix.go deleted file mode 100644 index 041d54c057..0000000000 --- a/vendor/github.com/containers/storage/lockfile_otherunix.go +++ /dev/null @@ -1,19 +0,0 @@ -// +build darwin freebsd - -package storage - -import ( - "time" - - "golang.org/x/sys/unix" -) - -func (l *lockfile) TouchedSince(when time.Time) bool { - st := unix.Stat_t{} - err := unix.Fstat(int(l.fd), &st) - if err != nil { - return true - } - touched := time.Unix(st.Mtimespec.Unix()) - return when.Before(touched) -} diff --git a/vendor/github.com/containers/storage/lockfile_unix.go b/vendor/github.com/containers/storage/lockfile_unix.go index 8e0f22cb5c..00215e928c 100644 --- a/vendor/github.com/containers/storage/lockfile_unix.go +++ b/vendor/github.com/containers/storage/lockfile_unix.go @@ -9,6 +9,7 @@ import ( "time" "github.com/containers/storage/pkg/stringid" + "github.com/containers/storage/pkg/system" "github.com/pkg/errors" "golang.org/x/sys/unix" ) @@ -25,6 +26,7 @@ type lockfile struct { locktype int16 locked bool ro bool + recursive bool } // openLock opens the file at path and returns the corresponding file @@ -75,7 +77,7 @@ func createLockerForPath(path string, ro bool) (Locker, error) { // lock locks the lockfile via FCTNL(2) based on the specified type and // command. -func (l *lockfile) lock(l_type int16) { +func (l *lockfile) lock(l_type int16, recursive bool) { lk := unix.Flock_t{ Type: l_type, Whence: int16(os.SEEK_SET), @@ -86,7 +88,13 @@ func (l *lockfile) lock(l_type int16) { case unix.F_RDLCK: l.rwMutex.RLock() case unix.F_WRLCK: - l.rwMutex.Lock() + if recursive { + // NOTE: that's okay as recursive is only set in RecursiveLock(), so + // there's no need to protect against hypothetical RDLCK cases. + l.rwMutex.RLock() + } else { + l.rwMutex.Lock() + } default: panic(fmt.Sprintf("attempted to acquire a file lock of unrecognized type %d", l_type)) } @@ -110,6 +118,7 @@ func (l *lockfile) lock(l_type int16) { } l.locktype = l_type l.locked = true + l.recursive = recursive l.counter++ } @@ -119,13 +128,24 @@ func (l *lockfile) Lock() { if l.ro { l.RLock() } else { - l.lock(unix.F_WRLCK) + l.lock(unix.F_WRLCK, false) + } +} + +// RecursiveLock locks the lockfile as a writer but allows for recursive +// acquisitions within the same process space. Note that RLock() will be called +// if it's a lockTypReader lock. +func (l *lockfile) RecursiveLock() { + if l.ro { + l.RLock() + } else { + l.lock(unix.F_WRLCK, true) } } // LockRead locks the lockfile as a reader. func (l *lockfile) RLock() { - l.lock(unix.F_RDLCK) + l.lock(unix.F_RDLCK, false) } // Unlock unlocks the lockfile. @@ -161,7 +181,7 @@ func (l *lockfile) Unlock() { // Close the file descriptor on the last unlock. unix.Close(int(l.fd)) } - if l.locktype == unix.F_RDLCK { + if l.locktype == unix.F_RDLCK || l.recursive { l.rwMutex.RUnlock() } else { l.rwMutex.Unlock() @@ -232,3 +252,14 @@ func (l *lockfile) Modified() (bool, error) { func (l *lockfile) IsReadWrite() bool { return !l.ro } + +// TouchedSince indicates if the lock file has been touched since the specified time +func (l *lockfile) TouchedSince(when time.Time) bool { + st, err := system.Fstat(int(l.fd)) + if err != nil { + return true + } + mtim := st.Mtim() + touched := time.Unix(mtim.Unix()) + return when.Before(touched) +} diff --git a/vendor/github.com/containers/storage/lockfile_windows.go b/vendor/github.com/containers/storage/lockfile_windows.go index c02069495c..caf7c184ad 100644 --- a/vendor/github.com/containers/storage/lockfile_windows.go +++ b/vendor/github.com/containers/storage/lockfile_windows.go @@ -36,6 +36,12 @@ func (l *lockfile) Lock() { l.locked = true } +func (l *lockfile) RecursiveLock() { + // We don't support Windows but a recursive writer-lock in one process-space + // is really a writer lock, so just panic. + panic("not supported") +} + func (l *lockfile) RLock() { l.mu.Lock() l.locked = true diff --git a/vendor/github.com/containers/storage/pkg/chrootarchive/archive.go b/vendor/github.com/containers/storage/pkg/chrootarchive/archive.go index a36ff1cb10..33ba6a128a 100644 --- a/vendor/github.com/containers/storage/pkg/chrootarchive/archive.go +++ b/vendor/github.com/containers/storage/pkg/chrootarchive/archive.go @@ -1,7 +1,7 @@ package chrootarchive import ( - "archive/tar" + stdtar "archive/tar" "fmt" "io" "io/ioutil" @@ -34,18 +34,34 @@ func NewArchiverWithChown(tarIDMappings *idtools.IDMappings, chownOpts *idtools. // The archive may be compressed with one of the following algorithms: // identity (uncompressed), gzip, bzip2, xz. func Untar(tarArchive io.Reader, dest string, options *archive.TarOptions) error { - return untarHandler(tarArchive, dest, options, true) + return untarHandler(tarArchive, dest, options, true, dest) +} + +// UntarWithRoot is the same as `Untar`, but allows you to pass in a root directory +// The root directory is the directory that will be chrooted to. +// `dest` must be a path within `root`, if it is not an error will be returned. +// +// `root` should set to a directory which is not controlled by any potentially +// malicious process. +// +// This should be used to prevent a potential attacker from manipulating `dest` +// such that it would provide access to files outside of `dest` through things +// like symlinks. Normally `ResolveSymlinksInScope` would handle this, however +// sanitizing symlinks in this manner is inherrently racey: +// ref: CVE-2018-15664 +func UntarWithRoot(tarArchive io.Reader, dest string, options *archive.TarOptions, root string) error { + return untarHandler(tarArchive, dest, options, true, root) } // UntarUncompressed reads a stream of bytes from `archive`, parses it as a tar archive, // and unpacks it into the directory at `dest`. // The archive must be an uncompressed stream. func UntarUncompressed(tarArchive io.Reader, dest string, options *archive.TarOptions) error { - return untarHandler(tarArchive, dest, options, false) + return untarHandler(tarArchive, dest, options, false, dest) } // Handler for teasing out the automatic decompression -func untarHandler(tarArchive io.Reader, dest string, options *archive.TarOptions, decompress bool) error { +func untarHandler(tarArchive io.Reader, dest string, options *archive.TarOptions, decompress bool, root string) error { if tarArchive == nil { return fmt.Errorf("Empty archive") } @@ -77,7 +93,15 @@ func untarHandler(tarArchive io.Reader, dest string, options *archive.TarOptions r = decompressedArchive } - return invokeUnpack(r, dest, options) + return invokeUnpack(r, dest, options, root) +} + +// Tar tars the requested path while chrooted to the specified root. +func Tar(srcPath string, options *archive.TarOptions, root string) (io.ReadCloser, error) { + if options == nil { + options = &archive.TarOptions{} + } + return invokePack(srcPath, options, root) } // CopyFileWithTarAndChown returns a function which copies a single file from outside @@ -99,7 +123,7 @@ func CopyFileWithTarAndChown(chownOpts *idtools.IDPair, hasher io.Writer, uidmap var hashWorker sync.WaitGroup hashWorker.Add(1) go func() { - t := tar.NewReader(contentReader) + t := stdtar.NewReader(contentReader) _, err := t.Next() if err != nil { hashError = err diff --git a/vendor/github.com/containers/storage/pkg/chrootarchive/archive_unix.go b/vendor/github.com/containers/storage/pkg/chrootarchive/archive_unix.go index e04ed787c0..ca9fb10d78 100644 --- a/vendor/github.com/containers/storage/pkg/chrootarchive/archive_unix.go +++ b/vendor/github.com/containers/storage/pkg/chrootarchive/archive_unix.go @@ -10,10 +10,13 @@ import ( "io" "io/ioutil" "os" + "path/filepath" "runtime" + "strings" "github.com/containers/storage/pkg/archive" "github.com/containers/storage/pkg/reexec" + "github.com/pkg/errors" ) // untar is the entry-point for storage-untar on re-exec. This is not used on @@ -23,18 +26,28 @@ func untar() { runtime.LockOSThread() flag.Parse() - var options *archive.TarOptions + var options archive.TarOptions //read the options from the pipe "ExtraFiles" if err := json.NewDecoder(os.NewFile(3, "options")).Decode(&options); err != nil { fatal(err) } - if err := chroot(flag.Arg(0)); err != nil { + dst := flag.Arg(0) + var root string + if len(flag.Args()) > 1 { + root = flag.Arg(1) + } + + if root == "" { + root = dst + } + + if err := chroot(root); err != nil { fatal(err) } - if err := archive.Unpack(os.Stdin, "/", options); err != nil { + if err := archive.Unpack(os.Stdin, dst, &options); err != nil { fatal(err) } // fully consume stdin in case it is zero padded @@ -45,7 +58,10 @@ func untar() { os.Exit(0) } -func invokeUnpack(decompressedArchive io.Reader, dest string, options *archive.TarOptions) error { +func invokeUnpack(decompressedArchive io.Reader, dest string, options *archive.TarOptions, root string) error { + if root == "" { + return errors.New("must specify a root to chroot to") + } // We can't pass a potentially large exclude list directly via cmd line // because we easily overrun the kernel's max argument/environment size @@ -57,7 +73,21 @@ func invokeUnpack(decompressedArchive io.Reader, dest string, options *archive.T return fmt.Errorf("Untar pipe failure: %v", err) } - cmd := reexec.Command("storage-untar", dest) + if root != "" { + relDest, err := filepath.Rel(root, dest) + if err != nil { + return err + } + if relDest == "." { + relDest = "/" + } + if relDest[0] != '/' { + relDest = "/" + relDest + } + dest = relDest + } + + cmd := reexec.Command("storage-untar", dest, root) cmd.Stdin = decompressedArchive cmd.ExtraFiles = append(cmd.ExtraFiles, r) @@ -68,6 +98,7 @@ func invokeUnpack(decompressedArchive io.Reader, dest string, options *archive.T if err := cmd.Start(); err != nil { return fmt.Errorf("Untar error on re-exec cmd: %v", err) } + //write the options to the pipe for the untar exec to read if err := json.NewEncoder(w).Encode(options); err != nil { return fmt.Errorf("Untar json encode to pipe failed: %v", err) @@ -84,3 +115,92 @@ func invokeUnpack(decompressedArchive io.Reader, dest string, options *archive.T } return nil } + +func tar() { + runtime.LockOSThread() + flag.Parse() + + src := flag.Arg(0) + var root string + if len(flag.Args()) > 1 { + root = flag.Arg(1) + } + + if root == "" { + root = src + } + + if err := realChroot(root); err != nil { + fatal(err) + } + + var options archive.TarOptions + if err := json.NewDecoder(os.Stdin).Decode(&options); err != nil { + fatal(err) + } + + rdr, err := archive.TarWithOptions(src, &options) + if err != nil { + fatal(err) + } + defer rdr.Close() + + if _, err := io.Copy(os.Stdout, rdr); err != nil { + fatal(err) + } + + os.Exit(0) +} + +func invokePack(srcPath string, options *archive.TarOptions, root string) (io.ReadCloser, error) { + if root == "" { + return nil, errors.New("root path must not be empty") + } + + relSrc, err := filepath.Rel(root, srcPath) + if err != nil { + return nil, err + } + if relSrc == "." { + relSrc = "/" + } + if relSrc[0] != '/' { + relSrc = "/" + relSrc + } + + // make sure we didn't trim a trailing slash with the call to `Rel` + if strings.HasSuffix(srcPath, "/") && !strings.HasSuffix(relSrc, "/") { + relSrc += "/" + } + + cmd := reexec.Command("storage-tar", relSrc, root) + + errBuff := bytes.NewBuffer(nil) + cmd.Stderr = errBuff + + tarR, tarW := io.Pipe() + cmd.Stdout = tarW + + stdin, err := cmd.StdinPipe() + if err != nil { + return nil, errors.Wrap(err, "error getting options pipe for tar process") + } + + if err := cmd.Start(); err != nil { + return nil, errors.Wrap(err, "tar error on re-exec cmd") + } + + go func() { + err := cmd.Wait() + err = errors.Wrapf(err, "error processing tar file: %s", errBuff) + tarW.CloseWithError(err) + }() + + if err := json.NewEncoder(stdin).Encode(options); err != nil { + stdin.Close() + return nil, errors.Wrap(err, "tar json encode to pipe failed") + } + stdin.Close() + + return tarR, nil +} diff --git a/vendor/github.com/containers/storage/pkg/chrootarchive/archive_windows.go b/vendor/github.com/containers/storage/pkg/chrootarchive/archive_windows.go index 93fde42206..8a5c680b14 100644 --- a/vendor/github.com/containers/storage/pkg/chrootarchive/archive_windows.go +++ b/vendor/github.com/containers/storage/pkg/chrootarchive/archive_windows.go @@ -14,9 +14,16 @@ func chroot(path string) error { func invokeUnpack(decompressedArchive io.ReadCloser, dest string, - options *archive.TarOptions) error { + options *archive.TarOptions, root string) error { // Windows is different to Linux here because Windows does not support // chroot. Hence there is no point sandboxing a chrooted process to // do the unpack. We call inline instead within the daemon process. return archive.Unpack(decompressedArchive, longpath.AddPrefix(dest), options) } + +func invokePack(srcPath string, options *archive.TarOptions, root string) (io.ReadCloser, error) { + // Windows is different to Linux here because Windows does not support + // chroot. Hence there is no point sandboxing a chrooted process to + // do the pack. We call inline instead within the daemon process. + return archive.TarWithOptions(srcPath, options) +} diff --git a/vendor/github.com/containers/storage/pkg/chrootarchive/chroot_unix.go b/vendor/github.com/containers/storage/pkg/chrootarchive/chroot_unix.go index f9b5dece8c..83278ee505 100644 --- a/vendor/github.com/containers/storage/pkg/chrootarchive/chroot_unix.go +++ b/vendor/github.com/containers/storage/pkg/chrootarchive/chroot_unix.go @@ -4,9 +4,13 @@ package chrootarchive import "golang.org/x/sys/unix" -func chroot(path string) error { +func realChroot(path string) error { if err := unix.Chroot(path); err != nil { return err } return unix.Chdir("/") } + +func chroot(path string) error { + return realChroot(path) +} diff --git a/vendor/github.com/containers/storage/pkg/chrootarchive/init_unix.go b/vendor/github.com/containers/storage/pkg/chrootarchive/init_unix.go index 21cd879928..ea08135e4d 100644 --- a/vendor/github.com/containers/storage/pkg/chrootarchive/init_unix.go +++ b/vendor/github.com/containers/storage/pkg/chrootarchive/init_unix.go @@ -14,6 +14,7 @@ import ( func init() { reexec.Register("storage-applyLayer", applyLayer) reexec.Register("storage-untar", untar) + reexec.Register("storage-tar", tar) } func fatal(err error) { diff --git a/vendor/github.com/containers/storage/pkg/system/stat_unix.go b/vendor/github.com/containers/storage/pkg/system/stat_unix.go index 91c7d121cc..f9a1b48778 100644 --- a/vendor/github.com/containers/storage/pkg/system/stat_unix.go +++ b/vendor/github.com/containers/storage/pkg/system/stat_unix.go @@ -58,3 +58,15 @@ func Stat(path string) (*StatT, error) { } return fromStatT(s) } + +// Fstat takes an open file descriptor and returns +// a system.StatT type pertaining to that file. +// +// Throws an error if the file descriptor is invalid +func Fstat(fd int) (*StatT, error) { + s := &syscall.Stat_t{} + if err := syscall.Fstat(fd, s); err != nil { + return nil, err + } + return fromStatT(s) +} diff --git a/vendor/github.com/containers/storage/store.go b/vendor/github.com/containers/storage/store.go index 27b00f6fe7..d70848dc54 100644 --- a/vendor/github.com/containers/storage/store.go +++ b/vendor/github.com/containers/storage/store.go @@ -1197,18 +1197,20 @@ func (s *store) CreateContainer(id string, names []string, image, layer, metadat } imageID = cimage.ID - createMappedLayer := imageHomeStore == istore + if cimage.TopLayer != "" { + createMappedLayer := imageHomeStore == istore + ilayer, err := s.imageTopLayerForMapping(cimage, imageHomeStore, createMappedLayer, rlstore, lstores, idMappingsOptions) + if err != nil { + return nil, err + } + imageTopLayer = ilayer - ilayer, err := s.imageTopLayerForMapping(cimage, imageHomeStore, createMappedLayer, rlstore, lstores, idMappingsOptions) - if err != nil { - return nil, err - } - imageTopLayer = ilayer - if !options.HostUIDMapping && len(options.UIDMap) == 0 { - uidMap = ilayer.UIDMap - } - if !options.HostGIDMapping && len(options.GIDMap) == 0 { - gidMap = ilayer.GIDMap + if !options.HostUIDMapping && len(options.UIDMap) == 0 { + uidMap = ilayer.UIDMap + } + if !options.HostGIDMapping && len(options.GIDMap) == 0 { + gidMap = ilayer.GIDMap + } } } else { rlstore.Lock() @@ -3396,12 +3398,18 @@ func init() { ReloadConfigurationFile(defaultConfigFile, &defaultStoreOptions) } +// GetDefaultMountOptions returns the default mountoptions defined in container/storage func GetDefaultMountOptions() ([]string, error) { + return GetMountOptions(defaultStoreOptions.GraphDriverName, defaultStoreOptions.GraphDriverOptions) +} + +// GetMountOptions returns the mountoptions for the specified driver and graphDriverOptions +func GetMountOptions(driver string, graphDriverOptions []string) ([]string, error) { mountOpts := []string{ ".mountopt", - fmt.Sprintf("%s.mountopt", defaultStoreOptions.GraphDriverName), + fmt.Sprintf("%s.mountopt", driver), } - for _, option := range defaultStoreOptions.GraphDriverOptions { + for _, option := range graphDriverOptions { key, val, err := parsers.ParseKeyValueOpt(option) if err != nil { return nil, err diff --git a/vendor/github.com/containers/storage/utils.go b/vendor/github.com/containers/storage/utils.go index 6c9f163a35..fafaaab5eb 100644 --- a/vendor/github.com/containers/storage/utils.go +++ b/vendor/github.com/containers/storage/utils.go @@ -71,14 +71,16 @@ func ParseIDMapping(UIDMapSlice, GIDMapSlice []string, subUIDMap, subGIDMap stri // GetRootlessRuntimeDir returns the runtime directory when running as non root func GetRootlessRuntimeDir(rootlessUid int) (string, error) { runtimeDir := os.Getenv("XDG_RUNTIME_DIR") - if runtimeDir == "" { - tmpDir := fmt.Sprintf("/run/user/%d", rootlessUid) - st, err := system.Stat(tmpDir) - if err == nil && int(st.UID()) == os.Getuid() && st.Mode()&0700 == 0700 && st.Mode()&0066 == 0000 { - return tmpDir, nil - } + + if runtimeDir != "" { + return runtimeDir, nil + } + tmpDir := fmt.Sprintf("/run/user/%d", rootlessUid) + st, err := system.Stat(tmpDir) + if err == nil && int(st.UID()) == os.Getuid() && st.Mode()&0700 == 0700 && st.Mode()&0066 == 0000 { + return tmpDir, nil } - tmpDir := fmt.Sprintf("%s/%d", os.TempDir(), rootlessUid) + tmpDir = fmt.Sprintf("%s/%d", os.TempDir(), rootlessUid) if err := os.MkdirAll(tmpDir, 0700); err != nil { logrus.Errorf("failed to create %s: %v", tmpDir, err) } else {