API: add network hotplug

Fixes kata-containers#113 Refactor generate interface and route, add network hotplug interface Signed-off-by: Ruidong Cao <[email protected]>
caoruidong · May 7, 2018 · 2217d38 · 2217d38
1 parent 992c895
commit 2217d38
Show file tree

Hide file tree

Showing 7 changed files with 296 additions and 92 deletions.
diff --git a/vendor/github.com/intel/govmm/qemu/qmp.go b/vendor/github.com/intel/govmm/qemu/qmp.go
diff --git a/virtcontainers/agent.go b/virtcontainers/agent.go
@@ -172,4 +172,7 @@ type agent interface {
 	// This function should be called after hot adding vCPUs or Memory.
 	// cpus specifies the number of CPUs that were added and the agent should online
 	onlineCPUMem(cpus uint32) error
+
+	// hotSetupNetwork will tell the agent to add a new nic for an existed Sandbox.
+	hotPlugNetwork(endpoint Endpoint) error
 }
diff --git a/virtcontainers/hyperstart_agent.go b/virtcontainers/hyperstart_agent.go
@@ -802,6 +802,10 @@ func (h *hyper) onlineCPUMem(cpus uint32) error {
 	return nil
 }
 
+func (h *hyper) hotPlugNetwork(endpoint Endpoint) error {
+	return fmt.Errorf("hyper don't support hot plug network")
+}
+
 func (h *hyper) check() error {
 	// cc-agent does not support check
 	return nil

diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go
@@ -326,81 +326,84 @@ func (k *kataAgent) generateInterfacesAndRoutes(networkNS NetworkNamespace) ([]*
 	var ifaces []*grpc.Interface
 
 	for _, endpoint := range networkNS.Endpoints {
+		ifaces = append(ifaces, k.generateInterface(endpoint))
+		routes = append(routes, k.generateRoutes(endpoint)...)
+	}
+	return ifaces, routes, nil
+}
 
-		var ipAddresses []*grpc.IPAddress
-		for _, addr := range endpoint.Properties().Addrs {
-			// Skip IPv6 because not supported
-			if addr.IP.To4() == nil {
-				// Skip IPv6 because not supported
-				k.Logger().WithFields(logrus.Fields{
-					"unsupported-address-type": "ipv6",
-					"address":                  addr,
-				}).Warn("unsupported address")
-				continue
-			}
-			// Skip localhost interface
-			if addr.IP.IsLoopback() {
-				continue
-			}
-			netMask, _ := addr.Mask.Size()
-			ipAddress := grpc.IPAddress{
-				Family:  grpc.IPFamily_v4,
-				Address: addr.IP.String(),
-				Mask:    fmt.Sprintf("%d", netMask),
-			}
-			ipAddresses = append(ipAddresses, &ipAddress)
+func (k *kataAgent) generateInterface(endpoint Endpoint) *grpc.Interface {
+	var ipAddresses []*grpc.IPAddress
+	for _, addr := range endpoint.Properties().Addrs {
+		// Skip IPv6 because not supported
+		if addr.IP.To4() == nil {
+			k.Logger().WithFields(logrus.Fields{
+				"unsupported-address-type": "ipv6",
+				"address":                  addr,
+			}).Warn("unsupported address")
+			continue
 		}
-		ifc := grpc.Interface{
-			IPAddresses: ipAddresses,
-			Device:      endpoint.Name(),
-			Name:        endpoint.Name(),
-			Mtu:         uint64(endpoint.Properties().Iface.MTU),
-			HwAddr:      endpoint.HardwareAddr(),
+		// Skip localhost interface
+		if addr.IP.IsLoopback() {
+			continue
 		}
+		netMask, _ := addr.Mask.Size()
+		ipAddress := grpc.IPAddress{
+			Family:  grpc.IPFamily_v4,
+			Address: addr.IP.String(),
+			Mask:    fmt.Sprintf("%d", netMask),
+		}
+		ipAddresses = append(ipAddresses, &ipAddress)
+	}
+	return &grpc.Interface{
+		IPAddresses: ipAddresses,
+		Device:      endpoint.Name(),
+		Name:        endpoint.Name(),
+		Mtu:         uint64(endpoint.Properties().Iface.MTU),
+		HwAddr:      endpoint.HardwareAddr(),
+	}
+}
 
-		ifaces = append(ifaces, &ifc)
-
-		for _, route := range endpoint.Properties().Routes {
-			var r grpc.Route
-
-			if route.Dst != nil {
-				r.Dest = route.Dst.String()
+func (k *kataAgent) generateRoutes(endpoint Endpoint) []*grpc.Route {
+	var routes []*grpc.Route
+	for _, route := range endpoint.Properties().Routes {
+		var r grpc.Route
+		if route.Dst != nil {
+			r.Dest = route.Dst.String()
 
-				if route.Dst.IP.To4() == nil {
-					// Skip IPv6 because not supported
-					k.Logger().WithFields(logrus.Fields{
-						"unsupported-route-type": "ipv6",
-						"destination":            r.Dest,
-					}).Warn("unsupported route")
-					continue
-				}
+			if route.Dst.IP.To4() == nil {
+				// Skip IPv6 because not supported
+				k.Logger().WithFields(logrus.Fields{
+					"unsupported-route-type": "ipv6",
+					"destination":            r.Dest,
+				}).Warn("unsupported route")
+				continue
 			}
+		}
 
-			if route.Gw != nil {
-				gateway := route.Gw.String()
-
-				if route.Gw.To4() == nil {
-					// Skip IPv6 because is is not supported
-					k.Logger().WithFields(logrus.Fields{
-						"unsupported-route-type": "ipv6",
-						"gateway":                gateway,
-					}).Warn("unsupported route")
-					continue
-				}
-				r.Gateway = gateway
-			}
+		if route.Gw != nil {
+			gateway := route.Gw.String()
 
-			if route.Src != nil {
-				r.Source = route.Src.String()
+			if route.Gw.To4() == nil {
+				// Skip IPv6 because is is not supported
+				k.Logger().WithFields(logrus.Fields{
+					"unsupported-route-type": "ipv6",
+					"gateway":                gateway,
+				}).Warn("unsupported route")
+				continue
 			}
+			r.Gateway = gateway
+		}
 
-			r.Device = endpoint.Name()
-			r.Scope = uint32(route.Scope)
-			routes = append(routes, &r)
-
+		if route.Src != nil {
+			r.Source = route.Src.String()
 		}
+
+		r.Device = endpoint.Name()
+		r.Scope = uint32(route.Scope)
+		routes = append(routes, &r)
 	}
-	return ifaces, routes, nil
+	return routes
 }
 
 func (k *kataAgent) startSandbox(sandbox *Sandbox) error {
@@ -450,36 +453,11 @@ func (k *kataAgent) startSandbox(sandbox *Sandbox) error {
 	if err != nil {
 		return err
 	}
-	for _, ifc := range interfaces {
-		// send update interface request
-		ifcReq := &grpc.UpdateInterfaceRequest{
-			Interface: ifc,
-		}
-		resultingInterface, err := k.sendReq(ifcReq)
-		if err != nil {
-			k.Logger().WithFields(logrus.Fields{
-				"interface-requested": fmt.Sprintf("%+v", ifc),
-				"resulting-interface": fmt.Sprintf("%+v", resultingInterface),
-			}).WithError(err).Error("update interface request failed")
-			return err
-		}
+	if err := k.plugInterfaces(interfaces); err != nil {
+		return err
 	}
-
-	if routes != nil {
-		routesReq := &grpc.UpdateRoutesRequest{
-			Routes: &grpc.Routes{
-				Routes: routes,
-			},
-		}
-
-		resultingRoutes, err := k.sendReq(routesReq)
-		if err != nil {
-			k.Logger().WithFields(logrus.Fields{
-				"routes-requested": fmt.Sprintf("%+v", routes),
-				"resulting-routes": fmt.Sprintf("%+v", resultingRoutes),
-			}).WithError(err).Error("update routes request failed")
-			return err
-		}
+	if err := k.plugRoutes(routes); err != nil {
+		return err
 	}
 
 	sharedDir9pOptions = append(sharedDir9pOptions, fmt.Sprintf("msize=%d", sandbox.config.HypervisorConfig.Msize9p))
@@ -901,6 +879,60 @@ func (k *kataAgent) onlineCPUMem(cpus uint32) error {
 	return err
 }
 
+func (k *kataAgent) hotPlugNetwork(endpoint Endpoint) error {
+	if endpoint == nil {
+		return fmt.Errorf("endpoint can not be nil")
+	}
+
+	interfaces := []*grpc.Interface{k.generateInterface(endpoint)}
+	routes := k.generateRoutes(endpoint)
+	if err := k.plugInterfaces(interfaces); err != nil {
+		return err
+	}
+	if err := k.plugRoutes(routes); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (k *kataAgent) plugInterfaces(interfaces []*grpc.Interface) error {
+	for _, ifc := range interfaces {
+		// send update interface request
+		ifcReq := &grpc.UpdateInterfaceRequest{
+			Interface: ifc,
+		}
+		resultingInterface, err := k.sendReq(ifcReq)
+		if err != nil {
+			k.Logger().WithFields(logrus.Fields{
+				"interface-requested": fmt.Sprintf("%+v", ifc),
+				"resulting-interface": fmt.Sprintf("%+v", resultingInterface),
+			}).WithError(err).Error("update interface request failed")
+			return err
+		}
+	}
+	return nil
+}
+
+func (k *kataAgent) plugRoutes(routes []*grpc.Route) error {
+	if routes != nil {
+		routesReq := &grpc.UpdateRoutesRequest{
+			Routes: &grpc.Routes{
+				Routes: routes,
+			},
+		}
+
+		resultingRoutes, err := k.sendReq(routesReq)
+		if err != nil {
+			k.Logger().WithFields(logrus.Fields{
+				"routes-requested": fmt.Sprintf("%+v", routes),
+				"resulting-routes": fmt.Sprintf("%+v", resultingRoutes),
+			}).WithError(err).Error("update routes request failed")
+			return err
+		}
+	}
+	return nil
+}
+
 func (k *kataAgent) connect() error {
 	if k.client != nil {
 		return nil

diff --git a/virtcontainers/network.go b/virtcontainers/network.go
@@ -151,6 +151,7 @@ type Endpoint interface {
 	SetProperties(NetworkInfo)
 	Attach(hypervisor) error
 	Detach() error
+	HotAttach(hypervisor) error
 }
 
 // VirtualEndpoint gathers a network pair and its properties.
@@ -232,6 +233,21 @@ func (endpoint *VirtualEndpoint) Detach() error {
 	return xconnectVMNetwork(&(endpoint.NetPair), false)
 }
 
+// HotAttach for the virtual endpoint uses hot plug device
+func (endpoint *VirtualEndpoint) HotAttach(h hypervisor) error {
+	networkLogger().Info("Hot Attaching virtual endpoint")
+	if err := xconnectVMNetwork(&(endpoint.NetPair), true); err != nil {
+		networkLogger().WithError(err).Error("Error bridging virtual ep")
+		return err
+	}
+
+	if err := h.hotplugAddDevice(endpoint, netDev); err != nil {
+		networkLogger().WithError(err).Error("Error attach virtual ep")
+		return err
+	}
+	return nil
+}
+
 // Properties returns the properties of the interface.
 func (endpoint *VhostUserEndpoint) Properties() NetworkInfo {
 	return endpoint.EndpointProperties
@@ -283,6 +299,11 @@ func (endpoint *VhostUserEndpoint) Detach() error {
 	return nil
 }
 
+// HotAttach for vhostuser endpoint not supported yet
+func (endpoint *VhostUserEndpoint) HotAttach(h hypervisor) error {
+	return fmt.Errorf("VhostUserEndpoint don't support Hot attach")
+}
+
 // Create a vhostuser endpoint
 func createVhostUserEndpoint(netInfo NetworkInfo, socket string) (*VhostUserEndpoint, error) {
 
@@ -346,6 +367,11 @@ func (endpoint *PhysicalEndpoint) Detach() error {
 	return bindNICToHost(endpoint)
 }
 
+// HotAttach for physical endpoint not supported yet
+func (endpoint *PhysicalEndpoint) HotAttach(h hypervisor) error {
+	return fmt.Errorf("PhysicalEndpoint don't support Hot attach")
+}
+
 // EndpointType identifies the type of the network endpoint.
 type EndpointType string
 

diff --git a/virtcontainers/noop_agent.go b/virtcontainers/noop_agent.go
@@ -79,6 +79,11 @@ func (n *noopAgent) onlineCPUMem(cpus uint32) error {
 	return nil
 }
 
+// hotPlugNetwork is the Noop agent Pod hot plug network implementation. It does nothing.
+func (n *noopAgent) hotPlugNetwork(endpoint Endpoint) error {
+	return nil
+}
+
 // check is the Noop agent health checker. It does nothing.
 func (n *noopAgent) check() error {
 	return nil