Skip to content

Commit

Permalink
network: Use the number of CPUS to set the multiqueue size
Browse files Browse the repository at this point in the history 
Instead of using a default queue size of 8 for macvtap fds,
use the number of CPUs on the guest as the queue size.
This is the recommended approach. This also shown better
performance results.

Fixes clearcontainers#680

Signed-off-by: Archana Shinde <[email protected]>
  • Loading branch information
@amshinde
amshinde committed Sep 12, 2018
1 parent 2f552fb commit 63785ca
Show file tree
Hide file tree
Showing 2 changed files with 22 additions and 32 deletions.
50 changes: 20 additions & 30 deletions virtcontainers/network.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,6 @@ const (
defaultRouteLabel = "default"
defaultFilePerms = 0600
defaultQlen = 1500
defaultQueues = 8
)

// DNSInfo describes the DNS setup related to a network interface.
Expand Down Expand Up @@ -226,7 +225,7 @@ func networkLogger() *logrus.Entry {
func (endpoint *VirtualEndpoint) Attach(h hypervisor) error {
networkLogger().WithField("endpoint-type", "virtual").Info("Attaching endpoint")

if err := xconnectVMNetwork(&(endpoint.NetPair), true); err != nil {
if err := xconnectVMNetwork(&(endpoint.NetPair), true, h.hypervisorConfig().NumVCPUs); err != nil {
networkLogger().WithError(err).Error("Error bridging virtual endpoint")
return err
}
Expand All @@ -246,14 +245,14 @@ func (endpoint *VirtualEndpoint) Detach(netNsCreated bool, netNsPath string) err
networkLogger().WithField("endpoint-type", "virtual").Info("Detaching endpoint")

return doNetNS(netNsPath, func(_ ns.NetNS) error {
return xconnectVMNetwork(&(endpoint.NetPair), false)
return xconnectVMNetwork(&(endpoint.NetPair), false, 0)
})
}

// HotAttach for the virtual endpoint uses hot plug device
func (endpoint *VirtualEndpoint) HotAttach(h hypervisor) error {
networkLogger().Info("Hot attaching virtual endpoint")
if err := xconnectVMNetwork(&(endpoint.NetPair), true); err != nil {
if err := xconnectVMNetwork(&(endpoint.NetPair), true, h.hypervisorConfig().NumVCPUs); err != nil {
networkLogger().WithError(err).Error("Error bridging virtual ep")
return err
}
Expand All @@ -272,7 +271,7 @@ func (endpoint *VirtualEndpoint) HotDetach(h hypervisor, netNsCreated bool, netN
}
networkLogger().Info("Hot detaching virtual endpoint")
if err := doNetNS(netNsPath, func(_ ns.NetNS) error {
return xconnectVMNetwork(&(endpoint.NetPair), false)
return xconnectVMNetwork(&(endpoint.NetPair), false, 0)
}); err != nil {
networkLogger().WithError(err).Error("Error abridging virtual ep")
return err
Expand Down Expand Up @@ -641,7 +640,7 @@ func newNetwork(networkType NetworkModel) network {
}
}

func createLink(netHandle *netlink.Handle, name string, expectedLink netlink.Link) (netlink.Link, []*os.File, error) {
func createLink(netHandle *netlink.Handle, name string, expectedLink netlink.Link, numCPUs int) (netlink.Link, []*os.File, error) {
var newLink netlink.Link
var fds []*os.File

Expand All @@ -655,7 +654,7 @@ func createLink(netHandle *netlink.Handle, name string, expectedLink netlink.Lin
newLink = &netlink.Tuntap{
LinkAttrs: netlink.LinkAttrs{Name: name},
Mode: netlink.TUNTAP_MODE_TAP,
Queues: defaultQueues,
Queues: numCPUs,
Flags: netlink.TUNTAP_MULTI_QUEUE_DEFAULTS | netlink.TUNTAP_VNET_HDR,
}
case (&netlink.Macvtap{}).Type():
Expand Down Expand Up @@ -722,21 +721,21 @@ func getLinkByName(netHandle *netlink.Handle, name string, expectedLink netlink.
}

// The endpoint type should dictate how the connection needs to be made
func xconnectVMNetwork(netPair *NetworkInterfacePair, connect bool) error {
func xconnectVMNetwork(netPair *NetworkInterfacePair, connect bool, numCPUs uint32) error {
if netPair.NetInterworkingModel == NetXConnectDefaultModel {
netPair.NetInterworkingModel = DefaultNetInterworkingModel
}
switch netPair.NetInterworkingModel {
case NetXConnectBridgedModel:
netPair.NetInterworkingModel = NetXConnectBridgedModel
if connect {
return bridgeNetworkPair(netPair)
return bridgeNetworkPair(netPair, numCPUs)
}
return unBridgeNetworkPair(*netPair)
case NetXConnectMacVtapModel:
netPair.NetInterworkingModel = NetXConnectMacVtapModel
if connect {
return tapNetworkPair(netPair)
return tapNetworkPair(netPair, numCPUs)
}
return untapNetworkPair(*netPair)
case NetXConnectEnlightenedModel:
Expand Down Expand Up @@ -786,10 +785,10 @@ const linkRange = 0xFFFF // This will allow upto 2^16 containers
const linkRetries = 128 // The numbers of time we try to find a non conflicting index
const macvtapWorkaround = true

func createMacVtap(netHandle *netlink.Handle, name string, link netlink.Link) (taplink netlink.Link, err error) {
func createMacVtap(netHandle *netlink.Handle, name string, link netlink.Link, numCPUs int) (taplink netlink.Link, err error) {

if !macvtapWorkaround {
taplink, _, err = createLink(netHandle, name, link)
taplink, _, err = createLink(netHandle, name, link, numCPUs)
return
}

Expand All @@ -798,7 +797,7 @@ func createMacVtap(netHandle *netlink.Handle, name string, link netlink.Link) (t
for i := 0; i < linkRetries; i++ {
index := hostLinkOffset + (r.Int() & linkRange)
link.Attrs().Index = index
taplink, _, err = createLink(netHandle, name, link)
taplink, _, err = createLink(netHandle, name, link, numCPUs)
if err == nil {
break
}
Expand All @@ -825,7 +824,7 @@ func setIPs(link netlink.Link, addrs []netlink.Addr) error {
return nil
}

func tapNetworkPair(netPair *NetworkInterfacePair) error {
func tapNetworkPair(netPair *NetworkInterfacePair, numCPUs uint32) error {
netHandle, err := netlink.NewHandle()
if err != nil {
return err
Expand All @@ -848,7 +847,7 @@ func tapNetworkPair(netPair *NetworkInterfacePair) error {
ParentIndex: vethLinkAttrs.Index,
},
},
})
}, int(numCPUs))

if err != nil {
return fmt.Errorf("Could not create TAP interface: %s", err)
Expand Down Expand Up @@ -900,21 +899,12 @@ func tapNetworkPair(netPair *NetworkInterfacePair) error {

// Note: The underlying interfaces need to be up prior to fd creation.

// Setup the multiqueue fds to be consumed by QEMU as macvtap cannot
// be directly connected.
// Ideally we want
// netdev.FDs, err = createMacvtapFds(netdev.ID, int(config.SMP.CPUs))

// We do not have global context here, hence a manifest constant
// that matches our minimum vCPU configuration
// Another option is to defer this to ciao qemu library which does have
// global context but cannot handle errors when setting up the network
netPair.VMFds, err = createMacvtapFds(tapLink.Attrs().Index, defaultQueues)
netPair.VMFds, err = createMacvtapFds(tapLink.Attrs().Index, int(numCPUs))
if err != nil {
return fmt.Errorf("Could not setup macvtap fds %s: %s", netPair.TAPIface, err)
}

vhostFds, err := createVhostFds(defaultQueues)
vhostFds, err := createVhostFds(int(numCPUs))
if err != nil {
return fmt.Errorf("Could not setup vhost fds %s : %s", netPair.VirtIface.Name, err)
}
Expand All @@ -923,20 +913,20 @@ func tapNetworkPair(netPair *NetworkInterfacePair) error {
return nil
}

func bridgeNetworkPair(netPair *NetworkInterfacePair) error {
func bridgeNetworkPair(netPair *NetworkInterfacePair, numCPUs uint32) error {
netHandle, err := netlink.NewHandle()
if err != nil {
return err
}
defer netHandle.Delete()

tapLink, fds, err := createLink(netHandle, netPair.TAPIface.Name, &netlink.Tuntap{})
tapLink, fds, err := createLink(netHandle, netPair.TAPIface.Name, &netlink.Tuntap{}, int(numCPUs))
if err != nil {
return fmt.Errorf("Could not create TAP interface: %s", err)
}
netPair.VMFds = fds

vhostFds, err := createVhostFds(defaultQueues)
vhostFds, err := createVhostFds(int(numCPUs))
if err != nil {
return fmt.Errorf("Could not setup vhost fds %s : %s", netPair.VirtIface.Name, err)
}
Expand Down Expand Up @@ -970,7 +960,7 @@ func bridgeNetworkPair(netPair *NetworkInterfacePair) error {
}

mcastSnoop := false
bridgeLink, _, err := createLink(netHandle, netPair.Name, &netlink.Bridge{MulticastSnooping: &mcastSnoop})
bridgeLink, _, err := createLink(netHandle, netPair.Name, &netlink.Bridge{MulticastSnooping: &mcastSnoop}, int(numCPUs))
if err != nil {
return fmt.Errorf("Could not create bridge: %s", err)
}
Expand Down
4 changes: 2 additions & 2 deletions virtcontainers/qemu.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -848,7 +848,7 @@ func (q *qemu) hotplugNetDevice(drive VirtualEndpoint, op operation) error {
if op == addDevice {
switch drive.NetPair.NetInterworkingModel {
case NetXConnectBridgedModel:
if err := q.qmpMonitorCh.qmp.ExecuteNetdevAdd(q.qmpMonitorCh.ctx, "tap", drive.NetPair.Name, drive.NetPair.TAPIface.Name, "no", "no", defaultQueues); err != nil {
if err := q.qmpMonitorCh.qmp.ExecuteNetdevAdd(q.qmpMonitorCh.ctx, "tap", drive.NetPair.Name, drive.NetPair.TAPIface.Name, "no", "no", int(q.config.NumVCPUs)); err != nil {
return err
}
case NetXConnectMacVtapModel:
Expand All @@ -863,7 +863,7 @@ func (q *qemu) hotplugNetDevice(drive VirtualEndpoint, op operation) error {
return err
}
drive.PCIAddr = fmt.Sprintf("%02x/%s", bridge.Addr, addr)
if err = q.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(q.qmpMonitorCh.ctx, drive.NetPair.Name, devID, drive.NetPair.TAPIface.HardAddr, addr, bridge.ID, defaultQueues); err != nil {
if err = q.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(q.qmpMonitorCh.ctx, drive.NetPair.Name, devID, drive.NetPair.TAPIface.HardAddr, addr, bridge.ID, int(q.config.NumVCPUs)); err != nil {
return err
}
} else {
Expand Down

0 comments on commit 63785ca

Please sign in to comment.