Skip to content

Commit

Permalink
Make rootless-cni setup more robust
Browse files Browse the repository at this point in the history 
The rootless cni namespace needs a valid /etc/resolv.conf file. On some
distros is a symlink to somewhere under /run. Because the kernel will
follow the symlink before mounting, it is not possible to mount a file
at exactly /etc/resolv.conf. We have to ensure that the link target will
be available in the rootless cni mount ns.

Fixes containers#10855

Also fixed a bug in the /var/lib/cni directory lookup logic. It used
`filepath.Base` instead of `filepath.Dir` and thus looping infinitely.

Fixes containers#10857

[NO TESTS NEEDED]

Signed-off-by: Paul Holzinger <[email protected]>
  • Loading branch information
@Luap99
Luap99 committed Jul 6, 2021
1 parent 2681484 commit 2c7c679
Show file tree
Hide file tree
Showing 2 changed files with 83 additions and 26 deletions.
7 changes: 6 additions & 1 deletion contrib/cirrus/runner.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,7 @@ function _run_swagger() {
trap "rm -f $envvarsfile" EXIT # contains secrets
# Warning: These values must _not_ be quoted, podman will not remove them.
#shellcheck disable=SC2154
cat <<eof>>$envvarsfile
cat <<eof >>$envvarsfile
GCPJSON=$GCPJSON
GCPNAME=$GCPNAME
GCPPROJECT=$GCPPROJECT
Expand Down Expand Up @@ -336,6 +336,11 @@ msg "************************************************************"

# shellcheck disable=SC2154
if [[ "$PRIV_NAME" == "rootless" ]] && [[ "$UID" -eq 0 ]]; then
# Remove /var/lib/cni, it is not required for rootless cni.
# We have to test that it works without this directory.
# https://github.com/containers/podman/issues/10857
rm -rf /var/lib/cni

req_env_vars ROOTLESS_USER
msg "Re-executing runner through ssh as user '$ROOTLESS_USER'"
msg "************************************************************"
Expand Down
102 changes: 77 additions & 25 deletions libpod/networking_linux.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,13 +111,33 @@ type RootlessCNI struct {
lock lockfile.Locker
}

// getPath will join the given path to the rootless cni dir
func (r *RootlessCNI) getPath(path string) string {
return filepath.Join(r.dir, path)
}

// Do - run the given function in the rootless cni ns.
// It does not lock the rootlessCNI lock, the caller
// should only lock when needed, e.g. for cni operations.
func (r *RootlessCNI) Do(toRun func() error) error {
err := r.ns.Do(func(_ ns.NetNS) error {
// before we can run the given function
// we have to setup all mounts correctly

// create a new mount namespace
// this should happen inside the netns thread
// Before we can run the given function,
// we have to setup all mounts correctly.

// The order of the mounts is IMPORTANT.
// The idea of the extra mount ns is to make /run and /var/lib/cni writeable
// for the cni plugins but not affecting the podman user namespace.
// Because the plugins also need access to XDG_RUNTIME_DIR/netns some special setup is needed.

// The following bind mounts are needed
// 1. XDG_RUNTIME_DIR/netns -> XDG_RUNTIME_DIR/rootless-cni/XDG_RUNTIME_DIR/netns
// 2. /run/systemd -> XDG_RUNTIME_DIR/rootless-cni/run/systemd (only if it exists)
// 3. XDG_RUNTIME_DIR/rootless-cni/resolv.conf -> /etc/resolv.conf or XDG_RUNTIME_DIR/rootless-cni/run/symlink/target
// 4. XDG_RUNTIME_DIR/rootless-cni/var/lib/cni -> /var/lib/cni (if /var/lib/cni does not exists use the parent dir)
// 5. XDG_RUNTIME_DIR/rootless-cni/run -> /run

// Create a new mount namespace,
// this must happen inside the netns thread.
err := unix.Unshare(unix.CLONE_NEWNS)
if err != nil {
return errors.Wrapf(err, "cannot create a new mount namespace")
Expand All @@ -127,44 +147,66 @@ func (r *RootlessCNI) Do(toRun func() error) error {
if err != nil {
return errors.Wrap(err, "could not get network namespace directory")
}
newNetNsDir := filepath.Join(r.dir, netNsDir)
// mount the netns into the new run to keep them accessible
// otherwise cni setup will fail because it cannot access the netns files
newNetNsDir := r.getPath(netNsDir)
// 1. Mount the netns into the new run to keep them accessible.
// Otherwise cni setup will fail because it cannot access the netns files.
err = unix.Mount(netNsDir, newNetNsDir, "none", unix.MS_BIND|unix.MS_SHARED|unix.MS_REC, "")
if err != nil {
return errors.Wrap(err, "failed to mount netns directory for rootless cni")
}

// mount resolv.conf to make use of the host dns
err = unix.Mount(filepath.Join(r.dir, "resolv.conf"), "/etc/resolv.conf", "none", unix.MS_BIND, "")
if err != nil {
return errors.Wrap(err, "failed to mount resolv.conf for rootless cni")
}

// also keep /run/systemd if it exists
// many files are symlinked into this dir, for example /dev/log
// 2. Also keep /run/systemd if it exists.
// Many files are symlinked into this dir, for example /dev/log.
runSystemd := "/run/systemd"
_, err = os.Stat(runSystemd)
if err == nil {
newRunSystemd := filepath.Join(r.dir, runSystemd[1:])
newRunSystemd := r.getPath(runSystemd)
err = unix.Mount(runSystemd, newRunSystemd, "none", unix.MS_BIND|unix.MS_REC, "")
if err != nil {
return errors.Wrap(err, "failed to mount /run/systemd directory for rootless cni")
}
}

// cni plugins need access to /var/lib/cni and /run
// 3. On some distros /etc/resolv.conf is symlinked to somewhere under /run.
// Because the kernel will follow the symlink before mounting, it is not
// possible to mount a file at /etc/resolv.conf. We have to ensure that
// the link target will be available in the mount ns.
// see: https://github.com/containers/podman/issues/10855
resolvePath := "/etc/resolv.conf"
resolvePath, err = filepath.EvalSymlinks(resolvePath)
if err != nil {
return err
}
if strings.HasPrefix(resolvePath, "/run/") {
resolvePath = r.getPath(resolvePath)
err = os.MkdirAll(filepath.Dir(resolvePath), 0700)
if err != nil {
return errors.Wrap(err, "failed to create rootless-cni resolv.conf directory")
}
// we want to bind mount on this file so we have to create the file first
_, err = os.OpenFile(resolvePath, os.O_CREATE|os.O_RDONLY, 0700)
if err != nil {
return errors.Wrap(err, "failed to create rootless-cni resolv.conf file")
}
}
// mount resolv.conf to make use of the host dns
err = unix.Mount(r.getPath("resolv.conf"), resolvePath, "none", unix.MS_BIND, "")
if err != nil {
return errors.Wrap(err, "failed to mount resolv.conf for rootless cni")
}

// 4. CNI plugins need access to /var/lib/cni and /run
varDir := ""
varTarget := persistentCNIDir
// we can only mount to a target dir which exists, check /var/lib/cni recursively
// while we could always use /var there are cases where a user might store the cni
// configs under /var/custom and this would break
for {
if _, err := os.Stat(varTarget); err == nil {
varDir = filepath.Join(r.dir, strings.TrimPrefix(varTarget, "/"))
varDir = r.getPath(varTarget)
break
}
varTarget = filepath.Base(varTarget)
varTarget = filepath.Dir(varTarget)
if varTarget == "/" {
break
}
Expand All @@ -177,8 +219,9 @@ func (r *RootlessCNI) Do(toRun func() error) error {
if err != nil {
return errors.Wrapf(err, "failed to mount %s for rootless cni", varTarget)
}
runDir := filepath.Join(r.dir, "run")
// recursive mount to keep the netns mount

// 5. Mount the new prepared run dir to /run, it has to be recursive to keep the other bind mounts.
runDir := r.getPath("run")
err = unix.Mount(runDir, "/run", "none", unix.MS_BIND|unix.MS_REC, "")
if err != nil {
return errors.Wrap(err, "failed to mount /run for rootless cni")
Expand Down Expand Up @@ -221,7 +264,7 @@ func (r *RootlessCNI) Cleanup(runtime *Runtime) error {
// make sure the the cni results (cache) dir is empty
// libpod instances with another root dir are not covered by the check above
// this allows several libpod instances to use the same rootless cni ns
contents, err := ioutil.ReadDir(filepath.Join(r.dir, "var/lib/cni/results"))
contents, err := ioutil.ReadDir(r.getPath("var/lib/cni/results"))
if (err == nil && len(contents) == 0) || os.IsNotExist(err) {
logrus.Debug("Cleaning up rootless cni namespace")
err = netns.UnmountNS(r.ns)
Expand All @@ -233,7 +276,7 @@ func (r *RootlessCNI) Cleanup(runtime *Runtime) error {
if err != nil {
logrus.Error(err)
}
b, err := ioutil.ReadFile(filepath.Join(r.dir, "rootless-cni-slirp4netns.pid"))
b, err := ioutil.ReadFile(r.getPath("rootless-cni-slirp4netns.pid"))
if err == nil {
var i int
i, err = strconv.Atoi(string(b))
Expand Down Expand Up @@ -395,10 +438,15 @@ func (r *Runtime) GetRootlessCNINetNs(new bool) (*RootlessCNI, error) {
if err != nil {
return nil, err
}
conf, err = resolvconf.FilterResolvDNS(conf.Content, netOptions.enableIPv6, true)
if err != nil {
return nil, err
}
searchDomains := resolvconf.GetSearchDomains(conf.Content)
dnsOptions := resolvconf.GetOptions(conf.Content)
nameServers := resolvconf.GetNameservers(conf.Content)

_, err = resolvconf.Build(filepath.Join(cniDir, "resolv.conf"), []string{resolveIP.String()}, searchDomains, dnsOptions)
_, err = resolvconf.Build(filepath.Join(cniDir, "resolv.conf"), append([]string{resolveIP.String()}, nameServers...), searchDomains, dnsOptions)
if err != nil {
return nil, errors.Wrap(err, "failed to create rootless cni resolv.conf")
}
Expand Down Expand Up @@ -500,7 +548,9 @@ func (r *Runtime) setUpOCICNIPod(podNetwork ocicni.PodNetwork) ([]ocicni.NetResu
// rootlessCNINS is nil if we are root
if rootlessCNINS != nil {
// execute the cni setup in the rootless net ns
rootlessCNINS.lock.Lock()
err = rootlessCNINS.Do(setUpPod)
rootlessCNINS.lock.Unlock()
} else {
err = setUpPod()
}
Expand Down Expand Up @@ -712,7 +762,9 @@ func (r *Runtime) teardownOCICNIPod(podNetwork ocicni.PodNetwork) error {
// rootlessCNINS is nil if we are root
if rootlessCNINS != nil {
// execute the cni setup in the rootless net ns
rootlessCNINS.lock.Lock()
err = rootlessCNINS.Do(tearDownPod)
rootlessCNINS.lock.Unlock()
if err == nil {
err = rootlessCNINS.Cleanup(r)
}
Expand Down

0 comments on commit 2c7c679

Please sign in to comment.