Add Scalable Function support

This patch adds new functions to get parse and retrieve the Scalable Functions(SF). This patch also adds the following new helpers getNetDevicesFromPath - retrive network devices based on path GetPfPciFromVfPci - retrieves the parent PF GetNetDeviceFromAux - gets auxiliary device name GetPfPciFromAux - retrieve PF PCI and SF index GetSfIndexByAuxDev - gets SF device name (e.g 'mlx5_core.sf.2') Authored-by: Dmytro Linkin <[email protected]> Signed-off-by: Dmytro Linkin <[email protected]> Signed-off-by: Alin-Gabriel Serdean <[email protected]>
k8snetworkplumbingwg · Mar 10, 2022 · 3248449 · 3248449
1 parent 687eb09
commit 3248449
Show file tree

Hide file tree

Showing 3 changed files with 152 additions and 15 deletions.
diff --git a/sriovnet.go b/sriovnet.go
@@ -429,18 +429,30 @@ func GetVfIndexByPciAddress(vfPciAddress string) (int, error) {
 	return -1, fmt.Errorf("vf index for %s not found", vfPciAddress)
 }
 
-// GetNetDevicesFromPci gets a PCI address (e.g '0000:03:00.1') and
-// returns the correlate list of netdevices
-func GetNetDevicesFromPci(pciAddress string) ([]string, error) {
-	pciDir := filepath.Join(PciSysDir, pciAddress, "net")
-	_, err := utilfs.Fs.Stat(pciDir)
+// GetPfPciFromVfPci retrieves the parent PF PCI address of the provided VF PCI address in D:B:D.f format
+func GetPfPciFromVfPci(vfPciAddress string) (string, error) {
+	pfPath := filepath.Join(PciSysDir, vfPciAddress, "physfn")
+	pciDevDir, err := utilfs.Fs.Readlink(pfPath)
+	if err != nil {
+		return "", fmt.Errorf("failed to read physfn link, provided address may not be a VF. %v", err)
+	}
+
+	pf := path.Base(pciDevDir)
+	if pf == "" {
+		return pf, fmt.Errorf("could not find PF PCI Address")
+	}
+	return pf, err
+}
+
+func getNetDevicesFromPath(dir string) ([]string, error) {
+	_, err := utilfs.Fs.Stat(dir)
 	if err != nil {
-		return nil, fmt.Errorf("cannot get a network device with pci address %v %v", pciAddress, err)
+		return nil, fmt.Errorf("cannot get network devices from path %s: %v", dir, err)
 	}
 
-	netDevicesFiles, err := utilfs.Fs.ReadDir(pciDir)
+	netDevicesFiles, err := utilfs.Fs.ReadDir(dir)
 	if err != nil {
-		return nil, fmt.Errorf("failed to get network device name in %v %v", pciDir, err)
+		return nil, fmt.Errorf("failed to get network devices name in %s: %v", dir, err)
 	}
 
 	netDevices := make([]string, 0, len(netDevicesFiles))
@@ -450,15 +462,55 @@ func GetNetDevicesFromPci(pciAddress string) ([]string, error) {
 	return netDevices, nil
 }
 
-// GetPfPciFromVfPci retrieves the parent PF PCI address of the provided VF PCI address in D:B:D.f format
-func GetPfPciFromVfPci(vfPciAddress string) (string, error) {
-	pfPath := filepath.Join(PciSysDir, vfPciAddress, "physfn")
-	pciDevDir, err := utilfs.Fs.Readlink(pfPath)
+// GetNetDeviceFromAux gets auxiliary device name (e.g 'mlx5_core.sf.2') and
+// returns the correlate netdevice
+func GetNetDevicesFromAux(auxDev string) ([]string, error) {
+	auxDir := filepath.Join(AuxSysDir, auxDev, "net")
+	return getNetDevicesFromPath(auxDir)
+}
+
+// GetNetDevicesFromPci gets a PCI address (e.g '0000:03:00.1') and
+// returns the correlate list of netdevices
+func GetNetDevicesFromPci(pciAddress string) ([]string, error) {
+	pciDir := filepath.Join(PciSysDir, pciAddress, "net")
+	return getNetDevicesFromPath(pciDir)
+}
+
+// GetSfIndexByAuxDev gets a SF device name (e.g 'mlx5_core.sf.2') and
+// returns the correlate SF index.
+func GetSfIndexByAuxDev(auxDev string) (int, error) {
+	sfnumFile := filepath.Join(AuxSysDir, auxDev, "sfnum")
+	if _, err := utilfs.Fs.Stat(sfnumFile); err != nil {
+		return -1, fmt.Errorf("cannot get sfnum for %s device: %v", auxDev, err)
+	}
+
+	sfnumStr, err := utilfs.Fs.ReadFile(sfnumFile)
 	if err != nil {
-		return "", fmt.Errorf("failed to read physfn link, provided address may not be a VF. %v", err)
+		return -1, fmt.Errorf("cannot read sfnum file for %s device: %v", auxDev, err)
 	}
 
-	pf := path.Base(pciDevDir)
+	sfnum, err := strconv.Atoi(string(sfnumStr))
+	if err != nil {
+		return -1, err
+	}
+	return sfnum, nil
+}
+
+// GetPfPciFromAux retrieves the parent PF PCI address of the provided auxiliary device in D.T.f format
+func GetPfPciFromAux(auxDev string) (string, error) {
+	auxPath := filepath.Join(AuxSysDir, auxDev)
+	absoluteAuxPath, err := utilfs.Fs.Readlink(auxPath)
+	if err != nil {
+		return "", fmt.Errorf("failed to read auxiliary link, provided device ID may be not auxiliary device. %v", err)
+	}
+
+	parent := filepath.Dir(absoluteAuxPath)
+	pf := filepath.Base(parent)
+	if pf != "" && pf[0:4] != "0000" { // XXX regexp??
+		// it's a nested auxiliary device. repeat
+		parent = filepath.Dir(parent)
+		pf = filepath.Base(parent)
+	}
 	if pf == "" {
 		return pf, fmt.Errorf("could not find PF PCI Address")
 	}

diff --git a/sriovnet_helper.go b/sriovnet_helper.go
@@ -10,6 +10,7 @@ import (
 const (
 	NetSysDir        = "/sys/class/net"
 	PciSysDir        = "/sys/bus/pci/devices"
+	AuxSysDir        = "/sys/bus/auxiliary/devices"
 	pcidevPrefix     = "device"
 	netdevDriverDir  = "device/driver"
 	netdevUnbindFile = "unbind"

diff --git a/sriovnet_switchdev.go b/sriovnet_switchdev.go
@@ -44,6 +44,9 @@ var pfPortRepRegex = regexp.MustCompile(`^(?:c\d+)?pf(\d+)$`)
 // Regex that matches on VF representor port name
 var vfPortRepRegex = regexp.MustCompile(`^(?:c\d+)?pf(\d+)vf(\d+)$`)
 
+// Regex that matches on SF representor port name
+var sfPortRepRegex = regexp.MustCompile(`^(?:c\d+)?pf(\d+)sf(\d+)$`)
+
 func parsePortName(physPortName string) (pfRepIndex, vfRepIndex int, err error) {
 	pfRepIndex = -1
 	vfRepIndex = -1
@@ -69,6 +72,20 @@ func parsePortName(physPortName string) (pfRepIndex, vfRepIndex int, err error)
 	return pfRepIndex, vfRepIndex, err
 }
 
+func parseSfPortName(physPortName string) (int, error) {
+	matches := sfPortRepRegex.FindStringSubmatch(physPortName)
+	//nolint:gomnd
+	if len(matches) != 3 {
+		return -1, fmt.Errorf("failed to parse physPortName %s", physPortName)
+	}
+
+	sfRepIndex, err := strconv.Atoi(matches[2])
+	if err != nil {
+		return -1, err
+	}
+	return sfRepIndex, err
+}
+
 func isSwitchdev(netdevice string) bool {
 	swIDFile := filepath.Join(NetSysDir, netdevice, netdevPhysSwitchID)
 	physSwitchID, err := utilfs.Fs.ReadFile(swIDFile)
@@ -110,6 +127,17 @@ func GetUplinkRepresentor(pciAddress string) (string, error) {
 	return "", fmt.Errorf("uplink for %s not found", pciAddress)
 }
 
+// GetUplinkRepresentorFromAux gets auxiliary device name (e.g 'mlx5_core.sf.2') and
+// returns the uplink representor netdev name for device.
+func GetUplinkRepresentorFromAux(auxDev string) (string, error) {
+	pfPci, err := GetPfPciFromAux(auxDev)
+	if err != nil {
+		return "", fmt.Errorf("failed to find uplink PCI device: %v", err)
+	}
+
+	return GetUplinkRepresentor(pfPci)
+}
+
 func GetVfRepresentor(uplink string, vfIndex int) (string, error) {
 	swIDFile := filepath.Join(NetSysDir, uplink, netdevPhysSwitchID)
 	physSwitchID, err := utilfs.Fs.ReadFile(swIDFile)
@@ -152,6 +180,29 @@ func GetVfRepresentor(uplink string, vfIndex int) (string, error) {
 	return "", fmt.Errorf("failed to find VF representor for uplink %s", uplink)
 }
 
+func GetSfRepresentor(uplink string, sfnum int) (string, error) {
+	pfNetPath := filepath.Join(NetSysDir, uplink, "device", "net")
+	devices, err := utilfs.Fs.ReadDir(pfNetPath)
+	if err != nil {
+		return "", err
+	}
+
+	for _, device := range devices {
+		physPortNameStr, err := getNetDevPhysPortName(device.Name())
+		if err != nil {
+			continue
+		}
+		sfRepIndex, _ := parseSfPortName(physPortNameStr)
+		if err != nil {
+			continue
+		}
+		if sfRepIndex == sfnum {
+			return device.Name(), nil
+		}
+	}
+	return "", fmt.Errorf("failed to find SF representor for uplink %s", uplink)
+}
+
 func getNetDevPhysPortName(netDev string) (string, error) {
 	devicePortNameFile := filepath.Join(NetSysDir, netDev, netdevPhysPortName)
 	physPortName, err := utilfs.Fs.ReadFile(devicePortNameFile)
@@ -206,7 +257,7 @@ func GetVfRepresentorDPU(pfID, vfIndex string) (string, error) {
 	}
 
 	// map for easy search of expected VF rep port name.
-	// Note: no supoport for Multi-Chassis DPUs
+	// Note: no support for Multi-Chassis DPUs
 	expectedPhysPortNames := map[string]interface{}{
 		fmt.Sprintf("pf%svf%s", pfID, vfIndex):   nil,
 		fmt.Sprintf("c1pf%svf%s", pfID, vfIndex): nil,
@@ -226,6 +277,39 @@ func GetVfRepresentorDPU(pfID, vfIndex string) (string, error) {
 	return netdev, nil
 }
 
+// GetSfRepresentorDPU returns SF representor on DPU for a host SF identified by pfID and sfIndex
+func GetSfRepresentorDPU(pfID, sfIndex string) (string, error) {
+	// pfID should be 0 or 1
+	if pfID != "0" && pfID != "1" {
+		return "", fmt.Errorf("unexpected pfID(%s). It should be 0 or 1", pfID)
+	}
+
+	// sfIndex should be an unsinged integer provided as a decimal number
+	if _, err := strconv.ParseUint(sfIndex, 10, 32); err != nil {
+		return "", fmt.Errorf("unexpected sfIndex(%s). It should be an unsigned decimal number", sfIndex)
+	}
+
+	// map for easy search of expected VF rep port name.
+	// Note: no support for Multi-Chassis DPUs
+	expectedPhysPortNames := map[string]interface{}{
+		fmt.Sprintf("pf%ssf%s", pfID, sfIndex):   nil,
+		fmt.Sprintf("c1pf%ssf%s", pfID, sfIndex): nil,
+	}
+
+	netdev, err := findNetdevWithPortNameCriteria(func(portName string) bool {
+		// if phys port name == pf<pfIndex>sf<sfIndex> or c1pf<pfIndex>sf<sfIndex> we have a match
+		if _, ok := expectedPhysPortNames[portName]; ok {
+			return true
+		}
+		return false
+	})
+
+	if err != nil {
+		return "", fmt.Errorf("sf representor for pfID:%s, sfIndex:%s not found", pfID, sfIndex)
+	}
+	return netdev, nil
+}
+
 // GetRepresentorPortFlavour returns the representor port flavour
 // Note: this method does not support old representor names used by old kernels
 // e.g <vf_num> and will return PORT_FLAVOUR_UNKNOWN for such cases.