fix rootless port forwarding with network dis-/connect

The rootlessport forwarder requires a child IP to be set. This must be a
valid ip in the container network namespace. The problem is that after a
network disconnect and connect the eth0 ip changed. Therefore the
packages are dropped since the source ip does no longer exists in the
netns.
One solution is to set the child IP to 127.0.0.1, however this is a
security problem. [1]

To fix this we have to recreate the ports after network connect and
disconnect. To make this work the rootlessport process exposes a socket
where podman network connect/disconnect connect to and send to new child
IP to rootlessport. The rootlessport process will remove all ports and
recreate them with the new correct child IP.

Also bump rootlesskit to v0.14.3 to fix a race with RemovePort().

Fixes containers#10052

[1] https://nvd.nist.gov/vuln/detail/CVE-2021-20199

Signed-off-by: Paul Holzinger <[email protected]>

docs/source/markdown/podman-network-connect.1.md

@@ -10,8 +10,6 @@ podman\-network\-connect - Connect a container to a network
 Connects a container to a network. A container can be connected to a network by name or by ID.
 Once connected, the container can communicate with other containers in the same network.
 
-This command is not available for rootless users.
-
 ## OPTIONS
 #### **--alias**
 Add network-scoped alias for the container.  If the network is using the `dnsname` CNI plugin, these aliases

docs/source/markdown/podman-network-disconnect.1.md

@@ -7,9 +7,10 @@ podman\-network\-disconnect - Disconnect a container from a network
 **podman network disconnect** [*options*] network container
 
 ## DESCRIPTION
-Disconnects a container from a network.
+Disconnects a container from a network. A container can be disconnected from a network by name or by ID.
+If all networks are disconnected from the container, it will behave like a container created with `--network=none`
+and it will longer have network connectivity until a network is connected again.
 
-This command is not available for rootless users.
 
 ## OPTIONS
 #### **--force**, **-f**

docs/source/markdown/podman-network-reload.1.md

@@ -13,8 +13,6 @@ Rootfull Podman relies on iptables rules in order to provide network connectivit
 this happens for example with `firewall-cmd --reload`, the container loses network connectivity. This command restores
 the network connectivity.
 
-This command is not available for rootless users since rootless containers are not affected by such connectivity problems.
-
 ## OPTIONS
 #### **--all**, **-a**
 

go.mod

@@ -52,7 +52,7 @@ require (
 	github.com/opencontainers/selinux v1.8.2
 	github.com/pkg/errors v0.9.1
 	github.com/pmezard/go-difflib v1.0.0
-	github.com/rootless-containers/rootlesskit v0.14.2
+	github.com/rootless-containers/rootlesskit v0.14.3
 	github.com/sirupsen/logrus v1.8.1
 	github.com/spf13/cobra v1.2.1
 	github.com/spf13/pflag v1.0.5

go.sum

@@ -395,7 +395,7 @@ github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6
 github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/godbus/dbus/v5 v5.0.4 h1:9349emZab16e7zQvpmsbtjc18ykshndd8y2PG3sgJbA=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
-github.com/gofrs/flock v0.8.0/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
+github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
 github.com/gogo/googleapis v1.2.0/go.mod h1:Njal3psf3qN6dwBtQfUmBZh2ybovJ0tlu3o/AC7HYjU=
 github.com/gogo/googleapis v1.4.0/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
@@ -547,7 +547,6 @@ github.com/insomniacslk/dhcp v0.0.0-20210120172423-cc9239ac6294/go.mod h1:TKl4jN
 github.com/ishidawataru/sctp v0.0.0-20210226210310-f2269e66cdee h1:PAXLXk1heNZ5yokbMBpVLZQxo43wCZxRwl00mX+dd44=
 github.com/ishidawataru/sctp v0.0.0-20210226210310-f2269e66cdee/go.mod h1:co9pwDoBCm1kGxawmb4sPq0cSIOOWNPT4KnHotMP1Zg=
 github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA=
-github.com/jamescun/tuntap v0.0.0-20190712092105-cb1fb277045c/go.mod h1:zzwpsgcYhzzIP5WyF8g9ivCv38cY9uAV9Gu0m3lThhE=
 github.com/jinzhu/copier v0.3.2 h1:QdBOCbaouLDYaIPFfi1bKv5F5tPpeTwXe4sD0jqtz5w=
 github.com/jinzhu/copier v0.3.2/go.mod h1:24xnZezI2Yqac9J61UC6/dG/k76ttpq0DdJI3QmUvro=
 github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
@@ -802,8 +801,8 @@ github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
-github.com/rootless-containers/rootlesskit v0.14.2 h1:jmsSyNyRG0QdWc3usppt5jEy5qOheeUsIINcymPrOFg=
-github.com/rootless-containers/rootlesskit v0.14.2/go.mod h1:nV3TpRISvwhZQSwo0nmQQnxjCxXr3mvrMi0oASLvzcg=
+github.com/rootless-containers/rootlesskit v0.14.3 h1:mS6lkZgT1McqUoZ9wjUIbYq7bWfd9aZGUgZgg8B55Sk=
+github.com/rootless-containers/rootlesskit v0.14.3/go.mod h1:Ai3detLzryb/4EkzXmNfh8aByUcBXp/qqkQusJs1SO8=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
@@ -829,6 +828,7 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1
 github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
+github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8/go.mod h1:P5HUIBuIWKbyjl083/loAegFkfbFNx5i2qEP4CNbm7E=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=

libpod/networking_linux.go

@@ -1214,7 +1214,29 @@ func (c *Container) NetworkDisconnect(nameOrID, netName string, force bool) erro
 		}
 	}
 	c.state.NetworkStatus = tmpNetworkStatus
-	return c.save()
+	err = c.save()
+	if err != nil {
+		return err
+	}
+
+	// OCICNI will set the loopback adpter down on teardown so we should set it up again
+	err = c.state.NetNS.Do(func(_ ns.NetNS) error {
+		link, err := netlink.LinkByName("lo")
+		if err != nil {
+			return err
+		}
+		err = netlink.LinkSetUp(link)
+		return err
+	})
+	if err != nil {
+		logrus.Warnf("failed to set loopback adpter up in the container: %v", err)
+	}
+	// Reload ports when there are still connected networks, maybe we removed the network interface with the child ip.
+	// Reloading without connected networks does not make sense, so we can skip this step.
+	if rootless.IsRootless() && len(tmpNetworkStatus) > 0 {
+		return c.reloadRootlessRLKPortMapping()
+	}
+	return nil
 }
 
 // ConnectNetwork connects a container to a given network
@@ -1306,7 +1328,16 @@ func (c *Container) NetworkConnect(nameOrID, netName string, aliases []string) e
 		networkStatus[index] = networkResults[0]
 		c.state.NetworkStatus = networkStatus
 	}
-	return c.save()
+	err = c.save()
+	if err != nil {
+		return err
+	}
+	// The first network needs a port reload to set the correct child ip for the rootlessport process.
+	// Adding a second network does not require a port reload because the child ip is still valid.
+	if rootless.IsRootless() && len(networks) == 0 {
+		return c.reloadRootlessRLKPortMapping()
+	}
+	return nil
 }
 
 // DisconnectContainerFromNetwork removes a container from its CNI network

libpod/networking_slirp4netns.go

@@ -17,6 +17,7 @@ import (
 	"time"
 
 	"github.com/containers/podman/v3/pkg/errorhandling"
+	"github.com/containers/podman/v3/pkg/rootless"
 	"github.com/containers/podman/v3/pkg/rootlessport"
 	"github.com/containers/podman/v3/pkg/servicereaper"
 	"github.com/pkg/errors"
@@ -466,29 +467,16 @@ func (r *Runtime) setupRootlessPortMappingViaRLK(ctr *Container, netnsPath strin
 		}
 	}
 
-	slirp4netnsIP, err := GetSlirp4netnsIP(ctr.slirp4netnsSubnet)
-	if err != nil {
-		return errors.Wrapf(err, "failed to get slirp4ns ip")
-	}
-	childIP := slirp4netnsIP.String()
-outer:
-	for _, r := range ctr.state.NetworkStatus {
-		for _, i := range r.IPs {
-			ipv4 := i.Address.IP.To4()
-			if ipv4 != nil {
-				childIP = ipv4.String()
-				break outer
-			}
-		}
-	}
-
+	childIP := getRootlessPortChildIP(ctr)
 	cfg := rootlessport.Config{
-		Mappings:  ctr.config.PortMappings,
-		NetNSPath: netnsPath,
-		ExitFD:    3,
-		ReadyFD:   4,
-		TmpDir:    ctr.runtime.config.Engine.TmpDir,
-		ChildIP:   childIP,
+		Mappings:    ctr.config.PortMappings,
+		NetNSPath:   netnsPath,
+		ExitFD:      3,
+		ReadyFD:     4,
+		TmpDir:      ctr.runtime.config.Engine.TmpDir,
+		ChildIP:     childIP,
+		ContainerID: ctr.config.ID,
+		RootlessCNI: ctr.config.NetMode.IsBridge() && rootless.IsRootless(),
 	}
 	cfgJSON, err := json.Marshal(cfg)
 	if err != nil {
@@ -617,3 +605,62 @@ func (r *Runtime) setupRootlessPortMappingViaSlirp(ctr *Container, cmd *exec.Cmd
 	logrus.Debug("slirp4netns port-forwarding setup via add_hostfwd is ready")
 	return nil
 }
+
+func getRootlessPortChildIP(c *Container) string {
+	if c.config.NetMode.IsSlirp4netns() {
+		slirp4netnsIP, err := GetSlirp4netnsIP(c.slirp4netnsSubnet)
+		if err != nil {
+			return ""
+		}
+		return slirp4netnsIP.String()
+	}
+
+	for _, r := range c.state.NetworkStatus {
+		for _, i := range r.IPs {
+			ipv4 := i.Address.IP.To4()
+			if ipv4 != nil {
+				return ipv4.String()
+			}
+		}
+	}
+	return ""
+}
+
+// reloadRootlessRLKPortMapping will trigger a reload for the port mappings in the rootlessport process.
+// This should only be called by network connect/disconnect and only as rootless.
+func (c *Container) reloadRootlessRLKPortMapping() error {
+	childIP := getRootlessPortChildIP(c)
+	logrus.Debugf("reloading rootless ports for container %s, childIP is %s", c.config.ID, childIP)
+
+	var conn net.Conn
+	var err error
+	// try three times to connect to the socket, maybe it is not ready yet
+	for i := 0; i < 3; i++ {
+		conn, err = net.Dial("unix", filepath.Join(c.runtime.config.Engine.TmpDir, "rp", c.config.ID))
+		if err == nil {
+			break
+		}
+		time.Sleep(250 * time.Millisecond)
+	}
+	if err != nil {
+		// This is not a hard error for backwards compatibility. A container started
+		// with an old version did not created the rootlessport socket.
+		logrus.Warnf("Could not reload rootless port mappings, port forwarding may no longer work correctly: %v", err)
+		return nil
+	}
+	defer conn.Close()
+	enc := json.NewEncoder(conn)
+	err = enc.Encode(childIP)
+	if err != nil {
+		return errors.Wrap(err, "port reloading failed")
+	}
+	b, err := ioutil.ReadAll(conn)
+	if err != nil {
+		return errors.Wrap(err, "port reloading failed")
+	}
+	data := string(b)
+	if data != "OK" {
+		return errors.Errorf("port reloading failed: %s", data)
+	}
+	return nil
+}

pkg/rootlessport/rootlessport_linux.go

@@ -17,9 +17,11 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"net"
 	"os"
 	"os/exec"
 	"os/signal"
+	"path/filepath"
 
 	"github.com/containernetworking/plugins/pkg/ns"
 	"github.com/containers/storage/pkg/reexec"
@@ -43,12 +45,14 @@ const (
 // Config needs to be provided to the process via stdin as a JSON string.
 // stdin needs to be closed after the message has been written.
 type Config struct {
-	Mappings  []ocicni.PortMapping
-	NetNSPath string
-	ExitFD    int
-	ReadyFD   int
-	TmpDir    string
-	ChildIP   string
+	Mappings    []ocicni.PortMapping
+	NetNSPath   string
+	ExitFD      int
+	ReadyFD     int
+	TmpDir      string
+	ChildIP     string
+	ContainerID string
+	RootlessCNI bool
 }
 
 func init() {
@@ -126,6 +130,12 @@ func parent() error {
 		}
 	}()
 
+	socketDir := filepath.Join(cfg.TmpDir, "rp")
+	err = os.MkdirAll(socketDir, 0700)
+	if err != nil {
+		return err
+	}
+
 	// create the parent driver
 	stateDir, err := ioutil.TempDir(cfg.TmpDir, "rootlessport")
 	if err != nil {
@@ -231,6 +241,16 @@ outer:
 		return err
 	}
 
+	// we only need to have a socket to reload ports when we run under rootless cni
+	if cfg.RootlessCNI {
+		socket, err := net.Listen("unix", filepath.Join(socketDir, cfg.ContainerID))
+		if err != nil {
+			return err
+		}
+		defer socket.Close()
+		go serve(socket, driver)
+	}
+
 	// write and close ReadyFD (convention is same as slirp4netns --ready-fd)
 	logrus.Info("ready")
 	if _, err := readyW.Write([]byte("1")); err != nil {
@@ -248,6 +268,53 @@ outer:
 	return nil
 }
 
+func serve(listener net.Listener, pm rkport.Manager) {
+	for {
+		conn, err := listener.Accept()
+		if err != nil {
+			// we cannot log this error, stderr is already closed
+			continue
+		}
+		ctx := context.TODO()
+		err = handler(ctx, conn, pm)
+		if err != nil {
+			conn.Write([]byte(err.Error()))
+		} else {
+			conn.Write([]byte("OK"))
+		}
+		conn.Close()
+	}
+}
+
+func handler(ctx context.Context, conn io.Reader, pm rkport.Manager) error {
+	var childIP string
+	dec := json.NewDecoder(conn)
+	err := dec.Decode(&childIP)
+	if err != nil {
+		return errors.Wrap(err, "rootless port failed to decode ports")
+	}
+	portStatus, err := pm.ListPorts(ctx)
+	if err != nil {
+		return errors.Wrap(err, "rootless port failed to list ports")
+	}
+	for _, status := range portStatus {
+		err = pm.RemovePort(ctx, status.ID)
+		if err != nil {
+			return errors.Wrap(err, "rootless port failed to remove port")
+		}
+	}
+	// add the ports with the new child IP
+	for _, status := range portStatus {
+		// set the new child IP
+		status.Spec.ChildIP = childIP
+		_, err = pm.AddPort(ctx, status.Spec)
+		if err != nil {
+			return errors.Wrap(err, "rootless port failed to add port")
+		}
+	}
+	return nil
+}
+
 func exposePorts(pm rkport.Manager, portMappings []ocicni.PortMapping, childIP string) error {
 	ctx := context.TODO()
 	for _, i := range portMappings {