Remove 'weave expose' race in CNI plugin

net/veth.go

@@ -271,3 +271,21 @@ func subnets(addrs []netlink.Addr) map[string]struct{} {
 	}
 	return subnets
 }
+
+func ExposeNAT(ipnet net.IPNet) error {
+	ipt, err := iptables.New()
+	if err != nil {
+		return err
+	}
+	cidr := ipnet.String()
+	if err := ipt.AppendUnique("nat", "WEAVE", "-s", cidr, "-d", "224.0.0.0/4", "-j", "RETURN"); err != nil {
+		return err
+	}
+	if err := ipt.AppendUnique("nat", "WEAVE", "-d", cidr, "!", "-s", cidr, "-j", "MASQUERADE"); err != nil {
+		return err
+	}
+	if err := ipt.AppendUnique("nat", "WEAVE", "-s", cidr, "!", "-d", cidr, "-j", "MASQUERADE"); err != nil {
+		return err
+	}
+	return nil
+}

plugin/net/cni.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"net"
+	"syscall"
 
 	"github.com/appc/cni/pkg/ipam"
 	"github.com/appc/cni/pkg/skel"
@@ -83,6 +84,9 @@ func (c *CNIPlugin) CmdAdd(args *skel.CmdArgs) error {
 			if err := assignBridgeIP(conf.BrName, bridgeIPResult.IP4.IP); err != nil {
 				return fmt.Errorf("unable to assign IP address to bridge: %s", err)
 			}
+			if err := weavenet.ExposeNAT(bridgeIPResult.IP4.IP); err != nil {
+				return fmt.Errorf("unable to create NAT rules: %s", err)
+			}
 			bridgeIP = bridgeIPResult.IP4.IP.IP
 		} else if err != nil {
 			return err
@@ -150,6 +154,11 @@ func assignBridgeIP(bridgeName string, ipnet net.IPNet) error {
 		return err
 	}
 	if err := netlink.AddrAdd(link, &netlink.Addr{IPNet: &ipnet}); err != nil {
+		// Treat as non-error if this address is already there
+		// - maybe another copy of this program just added it
+		if err == syscall.Errno(syscall.EEXIST) {
+			return nil
+		}
 		return fmt.Errorf("failed to add IP address to %q: %v", bridgeName, err)
 	}
 	return nil

prog/weave-kube/launch.sh

@@ -18,17 +18,6 @@ fi
 
 echo 1 > /proc/sys/net/bridge/bridge-nf-call-iptables
 
-# Create CNI config, if not already there
-if [ ! -f /etc/cni/net.d/10-weave.conf ] ; then
-    mkdir -p /etc/cni/net.d
-    cat > /etc/cni/net.d/10-weave.conf <<EOF
-{
-    "name": "weave",
-    "type": "weave-net"
-}
-EOF
-fi
-
 SOURCE_BINARY=/usr/bin/weaveutil
 VERSION=$(/home/weave/weaver --version | sed -E 's/weave router (.*?)/\1/')
 PLUGIN="weave-plugin-$VERSION"
@@ -152,7 +141,15 @@ done
     reclaim_ips "_" $IPS
 done
 
-# Expose the weave network so host processes can communicate with pods
-/home/weave/weave --local expose $WEAVE_EXPOSE_IP
+# Create CNI config, if not already there
+if [ ! -f /etc/cni/net.d/10-weave.conf ] ; then
+    mkdir -p /etc/cni/net.d
+    cat > /etc/cni/net.d/10-weave.conf <<EOF
+{
+    "name": "weave",
+    "type": "weave-net"
+}
+EOF
+fi
 
 wait $WEAVE_PID

prog/weaveutil/expose.go

@@ -0,0 +1,23 @@
+package main
+
+import (
+	weavenet "github.com/weaveworks/weave/net"
+)
+
+func exposeNAT(args []string) error {
+	if len(args) < 1 {
+		cmdUsage("expose-nat", "<cidr>...")
+	}
+
+	cidrs, err := parseCIDRs(args)
+	if err != nil {
+		return err
+	}
+
+	for _, cidr := range cidrs {
+		if err := weavenet.ExposeNAT(*cidr); err != nil {
+			return err
+		}
+	}
+	return nil
+}

prog/weaveutil/main.go

@@ -34,6 +34,7 @@ func init() {
 		"list-netdevs":           listNetDevs,
 		"cni-net":                cniNet,
 		"cni-ipam":               cniIPAM,
+		"expose-nat":             exposeNAT,
 	}
 }
 

test/840_weave_kube_3_test.sh

@@ -44,6 +44,14 @@ wait_for_connections() {
 
 assert_raises wait_for_connections
 
+# Check we can ping between the Weave bridg IPs on each host
+HOST1EXPIP=$($SSH $HOST1 "weave expose")
+HOST2EXPIP=$($SSH $HOST2 "weave expose")
+HOST3EXPIP=$($SSH $HOST3 "weave expose")
+assert_raises "run_on $HOST1 $PING $HOST2EXPIP"
+assert_raises "run_on $HOST2 $PING $HOST1EXPIP"
+assert_raises "run_on $HOST3 $PING $HOST2EXPIP"
+
 # See if we can get some pods running that connect to the network
 run_on $HOST1 "kubectl run hello --image=weaveworks/hello-world --replicas=3"
 

weave

@@ -398,7 +398,13 @@ add_iptables_rule() {
     IPTABLES_TABLE="$1"
     shift 1
     if ! run_iptables -t $IPTABLES_TABLE -C "$@" >/dev/null 2>&1 ; then
-        run_iptables -t $IPTABLES_TABLE -A "$@" >/dev/null
+        ## Loop until we get an exit code other than "temporarily unavailable"
+        while true ; do
+            run_iptables -t $IPTABLES_TABLE -A "$@" >/dev/null && return 0
+            if [ $? != 4 ] ; then
+                return 1
+            fi
+        done
     fi
 }
 
@@ -407,7 +413,13 @@ insert_iptables_rule() {
     IPTABLES_TABLE="$1"
     shift 1
     if ! run_iptables -t $IPTABLES_TABLE -C "$@" >/dev/null 2>&1 ; then
-        run_iptables -t $IPTABLES_TABLE -I "$@" >/dev/null
+        ## Loop until we get an exit code other than "temporarily unavailable"
+        while true ; do
+            run_iptables -t $IPTABLES_TABLE -I "$@" >/dev/null && return 0
+            if [ $? != 4 ] ; then
+                return 1
+            fi
+        done
     fi
 }
 
@@ -661,14 +673,6 @@ expose_ip() {
     done
 }
 
-expose_nat() {
-    for CIDR in $ALL_CIDRS ; do
-        add_iptables_rule nat WEAVE -s $CIDR -d 224.0.0.0/4 -j RETURN
-        add_iptables_rule nat WEAVE -d $CIDR ! -s $CIDR -j MASQUERADE
-        add_iptables_rule nat WEAVE -s $CIDR ! -d $CIDR -j MASQUERADE
-    done
-}
-
 # create veth with ends $1-$2, and then invoke $3..., removing the
 # veth on failure. No-op of veth already exists.
 create_veth() {
@@ -2222,7 +2226,7 @@ EOF
         fi
         create_bridge --without-ethtool
         expose_ip
-        expose_nat
+        util_op expose-nat $ALL_CIDRS
         show_addrs $ALL_CIDRS
         ;;
     hide)