add k8s-pure cni

build/docker/galaxy/Dockerfile

@@ -21,5 +21,5 @@ LABEL description="This Dockerfile is written for galaxy"
 WORKDIR /root/
 RUN yum install -y iproute iptables ipset
 COPY host-local loopback /opt/cni/galaxy/bin/
-COPY galaxy-k8s-sriov galaxy-k8s-vlan galaxy-bridge galaxy-flannel galaxy-veth galaxy-sdn tke-route-eni /opt/cni/galaxy/bin/
+COPY galaxy-k8s-sriov galaxy-k8s-vlan galaxy-underlay-veth galaxy-bridge galaxy-flannel galaxy-veth galaxy-sdn tke-route-eni /opt/cni/galaxy/bin/
 COPY galaxy /usr/bin/

build/lib/build.sh

@@ -52,11 +52,12 @@ function build::galaxy() {
   echo "Building plugins"
 
   # build galaxy cni plugins
-  PLUGINS="${PKG}/cni/k8s-vlan ${PKG}/cni/sdn ${PKG}/cni/veth ${PKG}/cni/k8s-sriov"
+  PLUGINS="${PKG}/cni/k8s-vlan ${PKG}/cni/sdn ${PKG}/cni/veth ${PKG}/cni/k8s-sriov ${PKG}/cni/underlay/veth"
   for d in ${PLUGINS}; do
-    plugin=$(basename $d)
+    plugin_dir=${d#"${PKG}/cni/"}
+    plugin=${plugin_dir//"/"/"-"}
     echo "  " ${plugin}
-    go build -o ${BIN_DIR}/${BIN_PREFIX}-${plugin} ${GOBUILD_FLAGS} ${PKG}/cni/${plugin}
+    go build -o ${BIN_DIR}/${BIN_PREFIX}-${plugin} ${GOBUILD_FLAGS} ${d}
   done
 
   # build cni plugins (no galaxy prefix)

cni/k8s-vlan/k8s_vlan.go

@@ -143,7 +143,7 @@ func setupVlanDevice(result020s []*t020.Result, vlanIds []uint16, args *skel.Cmd
 				args.IfName = fmt.Sprintf("eth%d", ifIndex)
 			}
 		}
-		if err := utils.VethConnectsHostWithContainer(result020, args, bridgeName, suffix); err != nil {
+		if err := utils.VethConnectsHostWithContainer(result020, args, bridgeName, suffix, nil); err != nil {
 			return err
 		}
 		_ = utils.SendGratuitousARP(args.IfName, result020s[0].IP4.IP.IP.String(), args.Netns, d.GratuitousArpRequest)

cni/underlay/veth/veth.go

@@ -0,0 +1,102 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"net"
+	"runtime"
+
+	"github.com/containernetworking/cni/pkg/skel"
+	"github.com/containernetworking/cni/pkg/types"
+	t020 "github.com/containernetworking/cni/pkg/types/020"
+	"github.com/containernetworking/cni/pkg/version"
+	"github.com/vishvananda/netlink"
+	"tkestack.io/galaxy/cni/ipam"
+	"tkestack.io/galaxy/pkg/network"
+	"tkestack.io/galaxy/pkg/network/vlan"
+	"tkestack.io/galaxy/pkg/utils"
+)
+
+func init() {
+	// this ensures that main runs only on main thread (thread group leader).
+	// since namespace ops (unshare, setns) are done for a single thread, we
+	// must ensure that the goroutine does not jump from OS thread to thread
+	runtime.LockOSThread()
+}
+
+func main() {
+	skel.PluginMain(cmdAdd, cmdDel, version.Legacy)
+}
+
+func cmdDel(args *skel.CmdArgs) error {
+	if err := utils.DeleteAllVeth(args.Netns); err != nil {
+		return err
+	}
+	return nil
+}
+
+func cmdAdd(args *skel.CmdArgs) error {
+	conf := vlan.NetConf{}
+	if err := json.Unmarshal(args.StdinData, &conf); err != nil {
+		return fmt.Errorf("conf error: %v", err)
+	}
+	vlanIds, results, err := ipam.Allocate(conf.IPAM.Type, args)
+	if err != nil {
+		return fmt.Errorf("allocate failed: %v", err)
+	}
+	if err := utils.UnSetArpIgnore("all"); err != nil {
+		return err
+	}
+	if err := utils.EnableNonlocalBind(); err != nil {
+		return err
+	}
+	ifName := args.IfName
+	ifIndex := 0
+	for i := range vlanIds {
+		vlanId := vlanIds[i]
+		result, err := t020.GetResult(results[i])
+		if err != nil {
+			return fmt.Errorf("result convert failed: %v", err)
+		}
+		device := conf.Device
+		// fixme: make route configurable
+		if i != 0 {
+			result.IP4.Routes = []types.Route{{
+				Dst: net.IPNet{
+					IP:   result.IP4.IP.IP.Mask(result.IP4.IP.Mask),
+					Mask: result.IP4.IP.Mask,
+				},
+			}}
+		}
+		var masterDevice netlink.Link
+		if masterDevice, err = vlan.SetupVlanInPureMode(device, vlanId); err != nil {
+			return fmt.Errorf("failed setup vlan: %v", err)
+		}
+		suffix := ""
+		if i != 0 {
+			suffix = fmt.Sprintf("-%d", i+1)
+			ifIndex++
+			args.IfName = fmt.Sprintf("eth%d", ifIndex)
+			if args.IfName == ifName {
+				ifIndex++
+				args.IfName = fmt.Sprintf("eth%d", ifIndex)
+			}
+		}
+		v4Addr, err := netlink.AddrList(masterDevice, netlink.FAMILY_V4)
+		if err != nil {
+			return fmt.Errorf("error getting ipv4 address %v", err)
+		}
+		filteredAddr := network.FilterLoopbackAddr(v4Addr)
+		var src net.IP
+		if len(filteredAddr) > 0 {
+			src = filteredAddr[0].IP
+		}
+		if err := utils.VethConnectsHostWithContainer(result, args, "", suffix, src); err != nil {
+			return fmt.Errorf("veth connect failed: %v", err)
+		}
+		utils.SendGratuitousARP(masterDevice.Attrs().Name, result.IP4.IP.IP.String(), "", conf.GratuitousArpRequest)
+	}
+	args.IfName = ifName
+	result, _ := t020.GetResult(results[0])
+	return result.Print()
+}

cni/veth/veth.go

@@ -60,30 +60,7 @@ func addHostRoute(containerIP *net.IPNet, vethHostName string, src string) error
 		return err
 	}
 	s := net.ParseIP(src)
-	if err = netlink.RouteAdd(&netlink.Route{
-		LinkIndex: vethHost.Attrs().Index,
-		Scope:     netlink.SCOPE_LINK,
-		Dst:       containerIP,
-		Gw:        nil,
-		Src:       s,
-	}); err != nil {
-		if s != nil {
-			// compatible change for old kernel which does not support src option
-			if err1 := netlink.RouteAdd(&netlink.Route{
-				LinkIndex: vethHost.Attrs().Index,
-				Scope:     netlink.SCOPE_LINK,
-				Dst:       containerIP,
-				Gw:        nil,
-			}); err1 != nil {
-				return fmt.Errorf("failed to add route '%v dev %v for old linux kernel': %v. With src option err: %v",
-					containerIP, vethHostName, err1, err)
-			} else {
-				return nil
-			}
-		}
-		return fmt.Errorf("failed to add route '%v dev %v src %v': %v", containerIP, vethHostName, s.String(), err)
-	}
-	return nil
+	return utils.AddHostRoute(containerIP, vethHost, s)
 }
 
 // #lizard forgives

doc/galaxy-config.md

@@ -14,7 +14,8 @@ Galaxy support multiple default networks and configures them to pods according t
     {"name":"galaxy-flannel", "type":"galaxy-flannel", "delegate":{"type":"galaxy-veth"}, "subnetFile":"/run/flannel
 /subnet.env"},
     {"name":"galaxy-k8s-vlan", "type":"galaxy-k8s-vlan", "device":"eth1", "default_bridge_name": "br0"},
-    {"name": "galaxy-k8s-sriov", "type": "galaxy-k8s-sriov", "device": "eth1", "vf_num": 10}
+    {"name": "galaxy-k8s-sriov", "type": "galaxy-k8s-sriov", "device": "eth1", "vf_num": 10},
+    {"name":"galaxy-underlay-veth", "type":"galaxy-underlay-veth", "device":"eth1"}
   ],
   "DefaultNetworks": ["galaxy-flannel"],
   "ENIIPNetwork": "galaxy-k8s-vlan"

doc/supported-cnis.md

@@ -62,3 +62,25 @@ TKE route ENI CNI is a [Tencent cloud ENI](https://cloud.tencent.com/product/eni
 and policy routing on host to connect host veth pair with ENI eth device.
 
 TKE route ENI CNI gets POD IPs either from CNI Args `ipinfos=[{"ip":"192.168.0.68/26","vlan":2,"gateway":"192.168.0.65"}]` or ipam CNI plugin.
+
+## underlay-veth CNI
+
+Underlay-veth CNI is a underlay network plugin which creates a veth pair to connect host network namespace with container.
+
+Compare with Vlan CNI, underlay-veth cni will not create any bridge device and using host route rules for packet forwarding.
+
+Underlay-veth CNI gets POD IPs either from CNI Args `ipinfos=[{"ip":"192.168.0.68/26","vlan":2,"gateway":"192.168.0.65"}]` or ipam CNI plugin.
+
+Underlay-veth CNI supports configuring vlan for POD IPs, and support multi-vlanId in single node.
+
+Compare with vlan CNI, underlay-veth CNI will not create vlan devices for host. The OS administrator should pre-configure vlan devices(with ip address) if needed.
+
+This is the configuration of underlay-veth CNI.
+
+```golang
+type NetConf struct {
+	types.NetConf
+	// The device which has IDC ip address, eg. eth0
+	Device string `json:"device"`
+}
+```

e2e/helper/util.go

@@ -126,6 +126,21 @@ func SetupDummyDev(ifName, cidr string) error {
 	return nil
 }
 
+func SetupVlanDev(ifName, parent, cidr string, vlanID int) error {
+	if out, err := Command("ip", "link", "add", "link",parent, "name", ifName, "type", "vlan", "id", fmt.Sprintf("%d", vlanID)).CombinedOutput(); err != nil {
+		if !strings.HasPrefix(string(out), "RTNETLINK answers: File exists") {
+			return fmt.Errorf("failed to add link %s: %v, %s", ifName, err, string(out))
+		}
+	}
+	if out, err := Command("ip", "address", "add", cidr, "dev", ifName).CombinedOutput(); err != nil {
+		return fmt.Errorf("failed to add address %s to %s: %v, %s", cidr, ifName, err, out)
+	}
+	if out, err := Command("ip", "link", "set", ifName, "up").CombinedOutput(); err != nil {
+		return fmt.Errorf("failed to set up %s: %v, %s", ifName, err, out)
+	}
+	return nil
+}
+
 func IPInfo(cidr string, vlan uint16) (string, error) {
 	_, ipNet, err := net.ParseCIDR(cidr)
 	if err != nil {

e2e/underlay/veth/veth_suite_test.go

@@ -0,0 +1,13 @@
+package veth_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestUnderlayVeth(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Underlay-Veth Suite")
+}

e2e/underlay/veth/veth_test.go

@@ -0,0 +1,50 @@
+package veth
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"tkestack.io/galaxy/e2e/helper"
+	"tkestack.io/galaxy/pkg/utils"
+)
+
+var _ = Describe("galaxy-underlay-veth vlan test", func() {
+	cni := "galaxy-underlay-veth"
+	ifaceCidr := "192.168.0.66/26"
+	vlanCidr := "192.168.2.68/26"
+	containerCidr := "192.168.2.69/26"
+	containerId := helper.NewContainerId()
+
+	AfterEach(func() {
+		helper.CleanupNetNS()
+		helper.CleanupIFace("dummy0.2")
+		helper.CleanupDummy()
+	})
+	It("vlan", func() {
+		netConf := []byte(`{
+    "name": "myvlan",
+    "type": "galaxy-underlay-veth",
+    "device": "dummy0"
+}`)
+		Expect(helper.SetupDummyDev("dummy0", ifaceCidr)).NotTo(HaveOccurred())
+		Expect(helper.SetupVlanDev("dummy0.2", "dummy0", vlanCidr, 2)).NotTo(HaveOccurred())
+		argsStr, err := helper.IPInfo(containerCidr, 2)
+		Expect(err).NotTo(HaveOccurred())
+		nsPath := helper.CmdAdd(containerId, "", argsStr, cni,
+			`{"cniVersion":"0.2.0","ip4":{"ip":"192.168.2.69/26","gateway":"192.168.2.65","routes":[{"dst":"0.0.0.0/0"}]},"dns":{}}`, netConf)
+		_, err = helper.Ping("192.168.2.69")
+		Expect(err).NotTo(HaveOccurred())
+
+		err = (&helper.NetworkTopology{
+			LeaveDevices: []*helper.LinkDevice{
+				helper.NewLinkDevice(nil, utils.HostVethName(containerId, ""), "veth"),
+			},
+		}).Verify()
+		Expect(err).NotTo(HaveOccurred())
+
+		// check container iface topology, route, neigh, ip address is expected
+		helper.CheckContainerTopology(nsPath, containerCidr, "192.168.2.65")
+
+		// test DEL command
+		helper.CmdDel(containerId, cni, netConf)
+	})
+})

hack/build-native.sh

@@ -13,10 +13,11 @@ cd $ROOT
 PKG=tkestack.io/galaxy
 BIN_PREFIX="galaxy"
 # build galaxy cni plugins
-PLUGINS="$GOPATH/src/${PKG}/cni/k8s-vlan $GOPATH/src/${PKG}/cni/sdn $GOPATH/src/${PKG}/cni/veth $GOPATH/src/${PKG}/cni/k8s-sriov"
+PLUGINS="$GOPATH/src/${PKG}/cni/k8s-vlan $GOPATH/src/${PKG}/cni/sdn $GOPATH/src/${PKG}/cni/veth $GOPATH/src/${PKG}/cni/k8s-sriov $GOPATH/src/${PKG}/cni/underlay/veth"
 for d in $PLUGINS; do
 	if [ -d $d ]; then
-		plugin=$(basename $d)
+	    plugin_dir=${d#"$GOPATH/src/${PKG}/cni/"}
+		plugin=${plugin_dir//"/"/"-"}
 		echo "  " $plugin
 		go build -o bin/${BIN_PREFIX}-$plugin $flags ${PKG}/cni/$plugin
 	fi

hack/test.sh

@@ -11,6 +11,6 @@ echo PATH=$PATH
 export GOOS=linux
 go test -coverpkg $PKG/pkg/... -coverprofile=coverage.txt -covermode=atomic -v ./tools/... ./cmd/... ./cni/... ./pkg/...
 # go tool cover -func=coverage.txt
-for i in e2e/k8s-vlan e2e/veth e2e/cni-request; do
+for i in e2e/k8s-vlan e2e/veth e2e/cni-request e2e/underlay/veth; do
   ginkgo -v $i
 done

pkg/api/cniutil/cni.go

@@ -137,6 +137,9 @@ func CmdAdd(cmdArgs *skel.CmdArgs, networkInfos []*NetworkInfo) (types.Result, e
 	for idx, networkInfo := range networkInfos {
 		//append additional args from network info
 		cmdArgs.Args = strings.TrimRight(fmt.Sprintf("%s;%s", cmdArgs.Args, BuildCNIArgs(networkInfo.Args)), ";")
+		if result != nil {
+			networkInfo.Conf["prevResult"] = result
+		}
 		result, err = DelegateAdd(networkInfo.Conf, cmdArgs, networkInfo.IfName)
 		if err != nil {
 			//fail to add cni, then delete all established CNIs recursively

pkg/galaxy/galaxy.go

@@ -24,7 +24,6 @@ import (
 
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
 	glog "k8s.io/klog"
 	"tkestack.io/galaxy/pkg/api/docker"
@@ -150,19 +149,11 @@ func (g *Galaxy) Stop() error {
 }
 
 func (g *Galaxy) initk8sClient() {
-	var clientConfig *rest.Config
-	var err error
-	if g.Master != "" || g.KubeConf != "" {
-		clientConfig, err = clientcmd.BuildConfigFromFlags(g.Master, g.KubeConf)
-		if err != nil {
-			glog.Fatalf("Invalid client config: %v", err)
-		}
-	} else {
-		clientConfig, err = rest.InClusterConfig()
-		if err != nil {
-			glog.Fatalf("Init InClient config failed: %v", err)
-		}
+	clientConfig, err := clientcmd.BuildConfigFromFlags(g.Master, g.KubeConf)
+	if err != nil {
+		glog.Fatalf("Invalid client config: %v", err)
 	}
+
 	clientConfig.QPS = 1000.0
 	clientConfig.Burst = 2000
 	glog.Infof("QPS: %e, Burst: %d", clientConfig.QPS, clientConfig.Burst)