From 6c9c1bd87567c24d0b1326348d8b1368c174167f Mon Sep 17 00:00:00 2001 From: adrianc Date: Thu, 29 Jun 2023 16:49:02 +0300 Subject: [PATCH] Add GetAuxSFDevByPciAndSfIndex This method returns SF device name on auxiliary bus given parent PCI address and SF index. Signed-off-by: adrianc --- errors.go | 25 +++++++++++ sriovnet_aux.go | 34 +++++++++++++++ sriovnet_aux_test.go | 101 ++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 154 insertions(+), 6 deletions(-) create mode 100644 errors.go diff --git a/errors.go b/errors.go new file mode 100644 index 0000000..aa6ff7e --- /dev/null +++ b/errors.go @@ -0,0 +1,25 @@ +/* +Copyright 2023 NVIDIA CORPORATION & AFFILIATES + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package sriovnet + +import ( + "errors" +) + +var ( + ErrDeviceNotFound = errors.New("device not found") +) diff --git a/sriovnet_aux.go b/sriovnet_aux.go index a60061b..88df9ad 100644 --- a/sriovnet_aux.go +++ b/sriovnet_aux.go @@ -28,6 +28,10 @@ import ( utilfs "github.com/k8snetworkplumbingwg/sriovnet/pkg/utils/filesystem" ) +const ( + u32Mask = 0xffffffff +) + // GetNetDeviceFromAux gets auxiliary device name (e.g 'mlx5_core.sf.2') and // returns the correlate netdevice func GetNetDevicesFromAux(auxDev string) ([]string, error) { @@ -103,9 +107,39 @@ func GetAuxNetDevicesFromPci(pciAddr string) ([]string, error) { auxDevs := make([]string, 0) for _, file := range files { + if !file.IsDir() { + // auxiliary devices appear as directory here. + continue + } if auxiliaryDeviceRe.MatchString(file.Name()) { auxDevs = append(auxDevs, file.Name()) } } return auxDevs, nil } + +// GetAuxSFDevByPciAndSFIndex returns auxiliary SF device name which is associated with the given parent PCI address +// and SF index. returns error if an error occurred. returns ErrDeviceNotFound error if device is not found. +func GetAuxSFDevByPciAndSFIndex(pciAddress string, sfIndex uint32) (string, error) { + devs, err := GetAuxNetDevicesFromPci(pciAddress) + if err != nil { + return "", err + } + + for _, dev := range devs { + // skip non sf devices + if !strings.Contains(dev, ".sf.") { + continue + } + + idx, err := GetSfIndexByAuxDev(dev) + if err != nil || idx < 0 { + continue + } + + if uint32(idx&u32Mask) == sfIndex { + return dev, nil + } + } + return "", ErrDeviceNotFound +} diff --git a/sriovnet_aux_test.go b/sriovnet_aux_test.go index 5e90783..7bdbcf4 100644 --- a/sriovnet_aux_test.go +++ b/sriovnet_aux_test.go @@ -29,6 +29,33 @@ import ( utilfs "github.com/k8snetworkplumbingwg/sriovnet/pkg/utils/filesystem" ) +type auxDevContext struct { + parent string + sfNum string + name string +} + +// setupAuxDevEnv creates (fake) auxiliary devices, returns error if setup failed. +func setUpAuxDevEnv(t *testing.T, auxDevs []auxDevContext) { + var err error + + err = utilfs.Fs.MkdirAll(AuxSysDir, os.FileMode(0755)) + assert.NoError(t, err) + + for _, dev := range auxDevs { + auxDevPathPCI := filepath.Join(PciSysDir, dev.parent, dev.name) + auxDevPathAux := filepath.Join(AuxSysDir, dev.name) + + err = utilfs.Fs.MkdirAll(auxDevPathPCI, os.FileMode(0755)) + assert.NoError(t, err) + if dev.sfNum != "" { + err = utilfs.Fs.WriteFile(filepath.Join(auxDevPathPCI, "sfnum"), []byte(dev.sfNum), os.FileMode(0655)) + assert.NoError(t, err) + } + utilfs.Fs.Symlink(auxDevPathPCI, auxDevPathAux) + } +} + func TestGetNetDevicesFromAuxSuccess(t *testing.T) { teardown := setupFakeFs(t) defer teardown() @@ -186,10 +213,10 @@ func TestGetUplinkRepresentorFromAuxNoSuchDevice(t *testing.T) { assert.Equal(t, "", pf) } -func createPciDevicePaths(pciAddr string, dirs []string) { +func createPciDevicePaths(t *testing.T, pciAddr string, dirs []string) { for _, dir := range dirs { path := filepath.Join(PciSysDir, pciAddr, dir) - _ = utilfs.Fs.MkdirAll(path, os.FileMode(0755)) + assert.NoError(t, utilfs.Fs.MkdirAll(path, os.FileMode(0755))) } } @@ -199,9 +226,9 @@ func TestGetAuxNetDevicesFromPciSuccess(t *testing.T) { pciAddr := "0000:00:01.0" devs := []string{"foo.bar.0", "foo.bar.1", "foo.baz.0"} - createPciDevicePaths(pciAddr, devs) + createPciDevicePaths(t, pciAddr, devs) // create few regular directories - createPciDevicePaths(pciAddr, []string{"infiniband", "net"}) + createPciDevicePaths(t, pciAddr, []string{"infiniband", "net"}) auxDevs, err := GetAuxNetDevicesFromPci(pciAddr) assert.NoError(t, err) @@ -213,7 +240,7 @@ func TestGetAuxNetDevicesFromPciSuccessNoDevices(t *testing.T) { defer teardown() pciAddr := "0000:00:01.0" - createPciDevicePaths(pciAddr, []string{"infiniband", "net"}) + createPciDevicePaths(t, pciAddr, []string{"infiniband", "net"}) auxDevs, err := GetAuxNetDevicesFromPci(pciAddr) assert.NoError(t, err) @@ -232,9 +259,71 @@ func TestGetAuxNetDevicesFromPciFailureNotANetworkDevice(t *testing.T) { defer teardown() pciAddr := "0000:00:01.0" - createPciDevicePaths(pciAddr, []string{"infiniband"}) + createPciDevicePaths(t, pciAddr, []string{"infiniband"}) auxDevs, err := GetAuxNetDevicesFromPci(pciAddr) assert.Error(t, err) assert.Equal(t, auxDevs, []string(nil)) } + +func TestGetAuxSFDevByPciAndSFIndex(t *testing.T) { + teardown := setupFakeFs(t) + defer teardown() + + pciAddr := "0000:03:00.0" + devs := []auxDevContext{ + { + parent: pciAddr, + sfNum: "", + name: "mlx5_core.eth.0", + }, + { + parent: pciAddr, + sfNum: "", + name: "mlx5_core.eth-rep.0", + }, + { + parent: pciAddr, + sfNum: "123", + name: "mlx5_core.sf.3", + }, + } + setUpAuxDevEnv(t, devs) + createPciDevicePaths(t, pciAddr, []string{"infiniband", "net"}) + + device, err := GetAuxSFDevByPciAndSFIndex(pciAddr, 123) + assert.NoError(t, err) + assert.Equal(t, "mlx5_core.sf.3", device) +} + +func TestGetAuxSFDevByPciAndSFIndexSFIndexNotFound(t *testing.T) { + teardown := setupFakeFs(t) + defer teardown() + + pciAddr := "0000:03:00.0" + devs := []auxDevContext{ + { + parent: pciAddr, + sfNum: "123", + name: "mlx5_core.sf.3", + }, + } + setUpAuxDevEnv(t, devs) + createPciDevicePaths(t, pciAddr, []string{"infiniband", "net"}) + + device, err := GetAuxSFDevByPciAndSFIndex(pciAddr, 122) + assert.Error(t, err) + assert.Equal(t, ErrDeviceNotFound, err) + assert.Equal(t, "", device) +} + +func TestGetAuxSFDevByPciAndSFIndexPCIAddressNotFound(t *testing.T) { + teardown := setupFakeFs(t) + defer teardown() + + createPciDevicePaths(t, "0000:03:00.0", []string{"infiniband", "net"}) + + _, err := GetAuxSFDevByPciAndSFIndex("0000:04:00.0", 4) + assert.Error(t, err) + assert.NotEqual(t, ErrDeviceNotFound, err) +}