From 57a67a54c86383488946a8eb89dbe86fb38c1843 Mon Sep 17 00:00:00 2001
From: Vladimir Popov <vladimir.popov@xored.com>
Date: Mon, 26 Oct 2020 10:20:57 +0700
Subject: [PATCH 01/11] Rework pcifunction

Signed-off-by: Vladimir Popov <vladimir.popov@xored.com>
---
 pkg/sriov/pcifunction/function.go          | 35 +++--------
 pkg/sriov/pcifunction/physical_function.go | 73 ++++++++++++++--------
 2 files changed, 54 insertions(+), 54 deletions(-)

diff --git a/pkg/sriov/pcifunction/function.go b/pkg/sriov/pcifunction/function.go
index dd2faafa..0ae5bc9b 100644
--- a/pkg/sriov/pcifunction/function.go
+++ b/pkg/sriov/pcifunction/function.go
@@ -18,6 +18,7 @@ package pcifunction
 
 import (
 	"io/ioutil"
+	"path"
 	"path/filepath"
 	"strconv"
 
@@ -35,29 +36,10 @@ const (
 // Function describes Linux PCI function
 type Function struct {
 	address        string
-	kernelDriver   string
 	pciDevicesPath string
 	pciDriversPath string
 }
 
-func newFunction(pciAddress, pciDevicesPath, pciDriversPath string) (*Function, error) {
-	f := &Function{
-		address:        pciAddress,
-		pciDevicesPath: pciDevicesPath,
-		pciDriversPath: pciDriversPath,
-	}
-
-	switch kernelDriver, err := f.GetBoundDriver(); {
-	case err != nil:
-		return nil, err
-	case kernelDriver == "":
-		return nil, errors.Errorf("no driver bound found for the device: %v", pciAddress)
-	default:
-		f.kernelDriver = kernelDriver
-		return f, nil
-	}
-}
-
 // GetPCIAddress returns f PCI address
 func (f *Function) GetPCIAddress() string {
 	return f.address
@@ -65,7 +47,7 @@ func (f *Function) GetPCIAddress() string {
 
 // GetNetInterfaceName returns f net interface name
 func (f *Function) GetNetInterfaceName() (string, error) {
-	fInfos, err := ioutil.ReadDir(filepath.Join(f.pciDevicesPath, f.address, netInterfacesPath))
+	fInfos, err := ioutil.ReadDir(f.withDevicePath(netInterfacesPath))
 	if err != nil {
 		return "", errors.Wrapf(err, "failed to read net directory for the device: %v", f.address)
 	}
@@ -87,7 +69,7 @@ func (f *Function) GetNetInterfaceName() (string, error) {
 
 // GetIOMMUGroup returns f IOMMU group id
 func (f *Function) GetIOMMUGroup() (uint, error) {
-	stringIOMMUGroup, err := evalSymlinkAndGetBaseName(filepath.Join(f.pciDevicesPath, f.address, iommuGroup))
+	stringIOMMUGroup, err := evalSymlinkAndGetBaseName(f.withDevicePath(iommuGroup))
 	if err != nil {
 		return 0, errors.Wrapf(err, "error evaluating IOMMU group id for the device: %v", f.address)
 	}
@@ -99,11 +81,11 @@ func (f *Function) GetIOMMUGroup() (uint, error) {
 
 // GetBoundDriver returns driver name that is bound to f, if no driver bound, returns ""
 func (f *Function) GetBoundDriver() (string, error) {
-	if !isFileExists(filepath.Join(f.pciDevicesPath, f.address, boundDriverPath)) {
+	if !isFileExists(f.withDevicePath(boundDriverPath)) {
 		return "", nil
 	}
 
-	driver, err := evalSymlinkAndGetBaseName(filepath.Join(f.pciDevicesPath, f.address, boundDriverPath))
+	driver, err := evalSymlinkAndGetBaseName(f.withDevicePath(boundDriverPath))
 	if err != nil {
 		return "", errors.Wrapf(err, "error evaluating bound driver for the device: %v", f.address)
 	}
@@ -119,7 +101,7 @@ func (f *Function) BindDriver(driver string) error {
 	case boundDriver == driver:
 		return nil
 	case boundDriver != "":
-		unbindPath := filepath.Join(f.pciDevicesPath, f.address, boundDriverPath, unbindDriverPath)
+		unbindPath := filepath.Join(f.withDevicePath(boundDriverPath, unbindDriverPath))
 		if err := ioutil.WriteFile(unbindPath, []byte(f.address), 0); err != nil {
 			return errors.Wrapf(err, "failed to unbind driver from the device: %v", f.address)
 		}
@@ -136,7 +118,6 @@ func (f *Function) BindDriver(driver string) error {
 	return nil
 }
 
-// BindKernelDriver unbinds currently bound driver and binds the default driver to f
-func (f *Function) BindKernelDriver() error {
-	return f.BindDriver(f.kernelDriver)
+func (f *Function) withDevicePath(elem ...string) string {
+	return path.Join(append([]string{f.pciDevicesPath, f.address}, elem...)...)
 }
diff --git a/pkg/sriov/pcifunction/physical_function.go b/pkg/sriov/pcifunction/physical_function.go
index cc1ed7cd..0cf93ed1 100644
--- a/pkg/sriov/pcifunction/physical_function.go
+++ b/pkg/sriov/pcifunction/physical_function.go
@@ -45,7 +45,9 @@ var (
 
 // PhysicalFunction describes Linux PCI physical function
 type PhysicalFunction struct {
-	*Function
+	virtualFunctions []*Function
+
+	Function
 }
 
 // NewPhysicalFunction returns a new PhysicalFunction
@@ -69,36 +71,54 @@ func NewPhysicalFunction(pciAddress, pciDevicesPath, pciDriversPath string) (*Ph
 		return nil, errors.Errorf("PCI device is not SR-IOV capable: %v", bdfPCIAddress)
 	}
 
-	f, err := newFunction(bdfPCIAddress, pciDevicesPath, pciDriversPath)
-	if err != nil {
+	pf := &PhysicalFunction{
+		Function: Function{
+			address:        pciAddress,
+			pciDevicesPath: pciDevicesPath,
+			pciDriversPath: pciDriversPath,
+		},
+	}
+	if err := pf.createVirtualFunctions(); err != nil {
 		return nil, err
 	}
-
-	return &PhysicalFunction{f}, nil
+	if err := pf.loadVirtualFunctions(); err != nil {
+		return nil, err
+	}
+	return pf, nil
 }
 
-// GetVirtualFunctionsCapacity returns count of virtual functions that can be created for the pf
-func (pf *PhysicalFunction) GetVirtualFunctionsCapacity() (uint, error) {
-	return readUintFromFile(filepath.Join(pf.pciDevicesPath, pf.address, totalVFFile))
+// GetVirtualFunctions returns pf virtual functions
+func (pf *PhysicalFunction) GetVirtualFunctions() []*Function {
+	vfs := make([]*Function, len(pf.virtualFunctions))
+	copy(vfs, pf.virtualFunctions)
+	return vfs
 }
 
-// CreateVirtualFunctions initializes virtual functions for the pf
-// NOTE: should fail if virtual functions are already exist
-func (pf *PhysicalFunction) CreateVirtualFunctions(vfCount uint) error {
-	configuredVFFilePath := filepath.Join(pf.pciDevicesPath, pf.address, configuredVFFile)
-	err := ioutil.WriteFile(configuredVFFilePath, []byte(strconv.Itoa(int(vfCount))), 0)
+func (pf *PhysicalFunction) createVirtualFunctions() error {
+	switch vfsCount, err := readUintFromFile(pf.withDevicePath(configuredVFFile)); {
+	case err != nil:
+		return errors.Wrapf(err, "failed to get configured VFs number for the PCI device: %v", pf.address)
+	case vfsCount > 0:
+		return nil
+	}
+
+	vfsCount, err := ioutil.ReadFile(pf.withDevicePath(totalVFFile))
+	if err != nil {
+		return errors.Wrapf(err, "failed to get available VFs number for the PCI device: %v", pf.address)
+	}
+
+	err = ioutil.WriteFile(pf.withDevicePath(configuredVFFile), vfsCount, 0)
 	if err != nil {
-		return errors.Wrapf(err, "failed to create virtual functions for the device: %v", pf.address)
+		return errors.Wrapf(err, "failed to create VFs for the PCI device: %v", pf.address)
 	}
 
 	return nil
 }
 
-// GetVirtualFunctions returns all virtual functions discovered for the pf
-func (pf *PhysicalFunction) GetVirtualFunctions() ([]*Function, error) {
-	vfDirs, err := filepath.Glob(filepath.Join(pf.pciDevicesPath, pf.address, virtualFunctionPrefix+"*"))
+func (pf *PhysicalFunction) loadVirtualFunctions() error {
+	vfDirs, err := filepath.Glob(pf.withDevicePath(virtualFunctionPrefix + "*"))
 	if err != nil {
-		return nil, errors.Wrapf(err, "failed to find virtual function directories for the device: %v", pf.address)
+		return errors.Wrapf(err, "failed to find virtual function directories for the device: %v", pf.address)
 	}
 
 	sort.Slice(vfDirs, func(i, k int) bool {
@@ -107,23 +127,22 @@ func (pf *PhysicalFunction) GetVirtualFunctions() ([]*Function, error) {
 		return iVFNum < kVFNum
 	})
 
-	var fs []*Function
 	for _, vfDir := range vfDirs {
 		vfDirInfo, err := os.Lstat(vfDir)
 		if err != nil || vfDirInfo.Mode()&os.ModeSymlink == 0 {
-			return nil, errors.Wrapf(err, "invalid virtual function directory: %v", vfDir)
+			return errors.Wrapf(err, "invalid virtual function directory: %v", vfDir)
 		}
 
 		linkName, err := filepath.EvalSymlinks(vfDir)
 		if err != nil {
-			return nil, errors.Wrapf(err, "invalid virtual function directory: %v", vfDir)
+			return errors.Wrapf(err, "invalid virtual function directory: %v", vfDir)
 		}
 
-		f, err := newFunction(filepath.Base(linkName), pf.pciDevicesPath, pf.pciDriversPath)
-		if err != nil {
-			return nil, err
-		}
-		fs = append(fs, f)
+		pf.virtualFunctions = append(pf.virtualFunctions, &Function{
+			address:        filepath.Base(linkName),
+			pciDevicesPath: pf.pciDevicesPath,
+			pciDriversPath: pf.pciDriversPath,
+		})
 	}
-	return fs, nil
+	return nil
 }

From 8218a7ed34c3a2b00b9551d9604c4b97c27a6df8 Mon Sep 17 00:00:00 2001
From: Vladimir Popov <vladimir.popov@xored.com>
Date: Mon, 26 Oct 2020 10:37:29 +0700
Subject: [PATCH 02/11] Add pci.Pool

Signed-off-by: Vladimir Popov <vladimir.popov@xored.com>
---
 pkg/sriov/pci/pool.go               | 155 ++++++++++++++++++++++++++++
 pkg/sriov/sriovtest/pci_function.go |   3 +-
 2 files changed, 156 insertions(+), 2 deletions(-)
 create mode 100644 pkg/sriov/pci/pool.go

diff --git a/pkg/sriov/pci/pool.go b/pkg/sriov/pci/pool.go
new file mode 100644
index 00000000..72148e94
--- /dev/null
+++ b/pkg/sriov/pci/pool.go
@@ -0,0 +1,155 @@
+// Copyright (c) 2020 Doc.ai and/or its affiliates.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package pci provides utils to work with pcifunction.Function
+package pci
+
+import (
+	"github.com/pkg/errors"
+
+	"github.com/networkservicemesh/sdk-sriov/pkg/sriov"
+	"github.com/networkservicemesh/sdk-sriov/pkg/sriov/config"
+	"github.com/networkservicemesh/sdk-sriov/pkg/sriov/pcifunction"
+	"github.com/networkservicemesh/sdk-sriov/pkg/sriov/sriovtest"
+)
+
+const (
+	vfioDriver = "vfio-pci"
+)
+
+type pciFunction interface {
+	GetBoundDriver() (string, error)
+	BindDriver(driver string) error
+
+	sriov.PCIFunction
+}
+
+// Pool manages pcifunction.Function
+type Pool struct {
+	functions             map[string]*function // pciAddr -> *function
+	functionsByIOMMUGroup map[uint][]*function // iommuGroup -> []*function
+}
+
+type function struct {
+	function     pciFunction
+	kernelDriver string
+}
+
+// NewPool returns a new PCI Pool
+func NewPool(pciDevicesPath, pciDriversPath string, conf *config.Config) (*Pool, error) {
+	p := &Pool{
+		functions:             map[string]*function{},
+		functionsByIOMMUGroup: map[uint][]*function{},
+	}
+
+	for pfPCIAddr := range conf.PhysicalFunctions {
+		pf, err := pcifunction.NewPhysicalFunction(pfPCIAddr, pciDevicesPath, pciDriversPath)
+		if err != nil {
+			return nil, err
+		}
+
+		if err := p.addFunction(&pf.Function); err != nil {
+			return nil, err
+		}
+
+		for _, vf := range pf.GetVirtualFunctions() {
+			if err := p.addFunction(vf); err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	return p, nil
+}
+
+// NewTestPool returns a new PCI Pool for testing
+func NewTestPool(physicalFunctions map[string]*sriovtest.PCIPhysicalFunction, conf *config.Config) (*Pool, error) {
+	p := &Pool{
+		functions:             map[string]*function{},
+		functionsByIOMMUGroup: map[uint][]*function{},
+	}
+
+	for pfPCIAddr := range conf.PhysicalFunctions {
+		pf, ok := physicalFunctions[pfPCIAddr]
+		if !ok {
+			return nil, errors.Errorf("PF doesn't exist: %v", pfPCIAddr)
+		}
+
+		_ = p.addFunction(&pf.PCIFunction)
+
+		for _, vf := range pf.Vfs {
+			_ = p.addFunction(vf)
+		}
+	}
+
+	return p, nil
+}
+
+func (p *Pool) addFunction(pcif pciFunction) (err error) {
+	f := &function{
+		function: pcif,
+	}
+
+	f.kernelDriver, err = pcif.GetBoundDriver()
+	if err != nil {
+		return err
+	}
+	p.functions[pcif.GetPCIAddress()] = f
+
+	iommuGroup, err := pcif.GetIOMMUGroup()
+	if err != nil {
+		return err
+	}
+	p.functionsByIOMMUGroup[iommuGroup] = append(p.functionsByIOMMUGroup[iommuGroup], f)
+
+	return nil
+}
+
+// GetPCIFunction returns PCI function for the given PCI address
+func (p *Pool) GetPCIFunction(pciAddr string) (sriov.PCIFunction, error) {
+	f, err := p.find(pciAddr)
+	if err != nil {
+		return nil, err
+	}
+	return f.function, nil
+}
+
+// BindDriver binds selected IOMMU group to the given driver type
+func (p *Pool) BindDriver(iommuGroup uint, driverType sriov.DriverType) error {
+	for _, f := range p.functionsByIOMMUGroup[iommuGroup] {
+		switch driverType {
+		case sriov.KernelDriver:
+			if err := f.function.BindDriver(f.kernelDriver); err != nil {
+				return err
+			}
+		case sriov.VFIOPCIDriver:
+			if err := f.function.BindDriver(vfioDriver); err != nil {
+				return err
+			}
+		default:
+			return errors.Errorf("driver type is not supported: %v", driverType)
+		}
+	}
+	return nil
+}
+
+func (p *Pool) find(pciAddr string) (*function, error) {
+	f, ok := p.functions[pciAddr]
+	if !ok {
+		return nil, errors.Errorf("PCI function doesn't exist: %v", pciAddr)
+	}
+	return f, nil
+}
diff --git a/pkg/sriov/sriovtest/pci_function.go b/pkg/sriov/sriovtest/pci_function.go
index e8645573..806893fd 100644
--- a/pkg/sriov/sriovtest/pci_function.go
+++ b/pkg/sriov/sriovtest/pci_function.go
@@ -20,8 +20,7 @@ import "github.com/networkservicemesh/sdk-sriov/pkg/sriov"
 
 // PCIPhysicalFunction is a test data class for pcifunction.PhysicalFunction
 type PCIPhysicalFunction struct {
-	Capacity int            `yaml:"capacity"`
-	Vfs      []*PCIFunction `yaml:"vfs"`
+	Vfs []*PCIFunction `yaml:"vfs"`
 
 	PCIFunction
 }

From 93bff1a053c6eb3e66479f3a7e5fca0b977cd434 Mon Sep 17 00:00:00 2001
From: Vladimir Popov <vladimir.popov@xored.com>
Date: Mon, 26 Oct 2020 10:38:12 +0700
Subject: [PATCH 03/11] Rework config, add pci.UpdateConfig()

Signed-off-by: Vladimir Popov <vladimir.popov@xored.com>
---
 pkg/sriov/config/config.go      | 50 ++++++++++++++++++++++++++++-----
 pkg/sriov/config/config.yml     | 15 ++++++----
 pkg/sriov/config/config_test.go | 29 ++++++++++++++-----
 pkg/sriov/pci/update_config.go  | 45 +++++++++++++++++++++++++++++
 pkg/sriov/resource/config.yml   | 19 +++++++++----
 pkg/sriov/resource/pool.go      | 20 ++++++-------
 pkg/sriov/token/config.yml      | 12 +++++---
 pkg/sriov/token/pool.go         |  8 +++---
 8 files changed, 155 insertions(+), 43 deletions(-)
 create mode 100644 pkg/sriov/pci/update_config.go

diff --git a/pkg/sriov/config/config.go b/pkg/sriov/config/config.go
index 80f2e76e..833fb773 100644
--- a/pkg/sriov/config/config.go
+++ b/pkg/sriov/config/config.go
@@ -34,19 +34,55 @@ type Config struct {
 
 func (c *Config) String() string {
 	sb := &strings.Builder{}
-	_, _ = sb.WriteString("&{PhysicalFunctions:map[")
+	_, _ = sb.WriteString("&{")
+
+	_, _ = sb.WriteString("PhysicalFunctions:map[")
+	var strs []string
 	for k, physicalFunction := range c.PhysicalFunctions {
-		_, _ = sb.WriteString(fmt.Sprintf("%s:%+v ", k, physicalFunction))
+		strs = append(strs, fmt.Sprintf("%s:%+v", k, physicalFunction))
 	}
-	_, _ = sb.WriteString("]}")
+	_, _ = sb.WriteString(strings.Join(strs, " "))
+	_, _ = sb.WriteString("]")
+
+	_, _ = sb.WriteString("}")
 	return sb.String()
 }
 
-// PhysicalFunction contains physical function capabilities, available services domains and virtual functions IOMMU groups
+// PhysicalFunction contains physical function capabilities, available services domains and virtual functions
 type PhysicalFunction struct {
-	Capabilities     []string        `yaml:"capabilities"`
-	ServiceDomains   []string        `yaml:"serviceDomains"`
-	VirtualFunctions map[string]uint `yaml:"virtualFunctions"`
+	Capabilities     []string           `yaml:"capabilities"`
+	ServiceDomains   []string           `yaml:"serviceDomains"`
+	VirtualFunctions []*VirtualFunction `yaml:"virtualFunctions"`
+}
+
+func (pf *PhysicalFunction) String() string {
+	sb := &strings.Builder{}
+	_, _ = sb.WriteString("&{")
+
+	_, _ = sb.WriteString("Capabilities:[")
+	_, _ = sb.WriteString(strings.Join(pf.Capabilities, " "))
+	_, _ = sb.WriteString("]")
+
+	_, _ = sb.WriteString(" ServiceDomains:[")
+	_, _ = sb.WriteString(strings.Join(pf.ServiceDomains, " "))
+	_, _ = sb.WriteString("]")
+
+	_, _ = sb.WriteString(" VirtualFunctions:[")
+	var strs []string
+	for _, virtualFunction := range pf.VirtualFunctions {
+		strs = append(strs, fmt.Sprintf("%+v", virtualFunction))
+	}
+	_, _ = sb.WriteString(strings.Join(strs, " "))
+	_, _ = sb.WriteString("]")
+
+	_, _ = sb.WriteString("}")
+	return sb.String()
+}
+
+// VirtualFunction contains
+type VirtualFunction struct {
+	Address    string `yaml:"address"`
+	IOMMUGroup uint   `yaml:"iommuGroup"`
 }
 
 // ReadConfig reads configuration from file
diff --git a/pkg/sriov/config/config.yml b/pkg/sriov/config/config.yml
index fffa2553..2a3c0384 100644
--- a/pkg/sriov/config/config.yml
+++ b/pkg/sriov/config/config.yml
@@ -7,8 +7,10 @@ physicalFunctions:
     serviceDomains:
       - service.domain.1
     virtualFunctions:
-      0000:01:00.1: 1
-      0000:01:00.2: 2
+      - address: 0000:01:00.1
+        iommuGroup: 1
+      - address: 0000:01:00.2
+        iommuGroup: 2
   0000:02:00.0:
     capabilities:
       - intel
@@ -17,6 +19,9 @@ physicalFunctions:
       - service.domain.1
       - service.domain.2
     virtualFunctions:
-      0000:02:00.1: 1
-      0000:02:00.2: 2
-      0000:02:00.3: 3
+      - address: 0000:02:00.1
+        iommuGroup: 1
+      - address: 0000:02:00.2
+        iommuGroup: 2
+      - address: 0000:02:00.3
+        iommuGroup: 3
diff --git a/pkg/sriov/config/config_test.go b/pkg/sriov/config/config_test.go
index 9908706f..b8d78b34 100644
--- a/pkg/sriov/config/config_test.go
+++ b/pkg/sriov/config/config_test.go
@@ -54,9 +54,15 @@ func TestReadConfigFile(t *testing.T) {
 				ServiceDomains: []string{
 					serviceDomain1,
 				},
-				VirtualFunctions: map[string]uint{
-					vf11PciAddr: 1,
-					vf12PciAddr: 2,
+				VirtualFunctions: []*config.VirtualFunction{
+					{
+						Address:    vf11PciAddr,
+						IOMMUGroup: 1,
+					},
+					{
+						Address:    vf12PciAddr,
+						IOMMUGroup: 2,
+					},
 				},
 			},
 			pf2PciAddr: {
@@ -68,10 +74,19 @@ func TestReadConfigFile(t *testing.T) {
 					serviceDomain1,
 					serviceDomain2,
 				},
-				VirtualFunctions: map[string]uint{
-					vf21PciAddr: 1,
-					vf22PciAddr: 2,
-					vf23PciAddr: 3,
+				VirtualFunctions: []*config.VirtualFunction{
+					{
+						Address:    vf21PciAddr,
+						IOMMUGroup: 1,
+					},
+					{
+						Address:    vf22PciAddr,
+						IOMMUGroup: 2,
+					},
+					{
+						Address:    vf23PciAddr,
+						IOMMUGroup: 3,
+					},
 				},
 			},
 		},
diff --git a/pkg/sriov/pci/update_config.go b/pkg/sriov/pci/update_config.go
new file mode 100644
index 00000000..96238d0c
--- /dev/null
+++ b/pkg/sriov/pci/update_config.go
@@ -0,0 +1,45 @@
+// Copyright (c) 2020 Doc.ai and/or its affiliates.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pci
+
+import (
+	"github.com/networkservicemesh/sdk-sriov/pkg/sriov/config"
+	"github.com/networkservicemesh/sdk-sriov/pkg/sriov/pcifunction"
+)
+
+// UpdateConfig updates config with virtual functions
+func UpdateConfig(pciDevicesPath, pciDriversPath string, conf *config.Config) error {
+	for pfPCIAddr, pFun := range conf.PhysicalFunctions {
+		pf, err := pcifunction.NewPhysicalFunction(pfPCIAddr, pciDevicesPath, pciDriversPath)
+		if err != nil {
+			return err
+		}
+
+		for _, vf := range pf.GetVirtualFunctions() {
+			iommuGroup, err := vf.GetIOMMUGroup()
+			if err != nil {
+				return err
+			}
+
+			pFun.VirtualFunctions = append(pFun.VirtualFunctions, &config.VirtualFunction{
+				Address:    vf.GetPCIAddress(),
+				IOMMUGroup: iommuGroup,
+			})
+		}
+	}
+	return nil
+}
diff --git a/pkg/sriov/resource/config.yml b/pkg/sriov/resource/config.yml
index 231dcb32..ac391bfe 100644
--- a/pkg/sriov/resource/config.yml
+++ b/pkg/sriov/resource/config.yml
@@ -7,7 +7,8 @@ physicalFunctions:
     serviceDomains:
       - service.domain.1
     virtualFunctions:
-      0000:01:00.1: 1
+      - address: 0000:01:00.1
+        iommuGroup: 1
   0000:02:00.0:
     capabilities:
       - intel
@@ -15,8 +16,10 @@ physicalFunctions:
     serviceDomains:
       - service.domain.2
     virtualFunctions:
-      0000:02:00.1: 1
-      0000:02:00.2: 2
+      - address: 0000:02:00.1
+        iommuGroup: 1
+      - address: 0000:02:00.2
+        iommuGroup: 2
   0000:03:00.0:
     capabilities:
       - intel
@@ -24,6 +27,10 @@ physicalFunctions:
     serviceDomains:
       - service.domain.2
     virtualFunctions:
-      0000:03:00.1: 1
-      0000:03:00.2: 1
-      0000:03:00.3: 1
+      - address: 0000:03:00.1
+        iommuGroup: 1
+      - address: 0000:03:00.2
+        iommuGroup: 1
+      - address: 0000:03:00.3
+        iommuGroup: 1
+
diff --git a/pkg/sriov/resource/pool.go b/pkg/sriov/resource/pool.go
index 454f71bf..4cac47f0 100644
--- a/pkg/sriov/resource/pool.go
+++ b/pkg/sriov/resource/pool.go
@@ -68,30 +68,30 @@ func NewPool(tokenPool TokenPool, cfg *config.Config) *Pool {
 		tokenPool:         tokenPool,
 	}
 
-	for pfPCIAddr, pff := range cfg.PhysicalFunctions {
+	for pfPCIAddr, pFun := range cfg.PhysicalFunctions {
 		pf := &physicalFunction{
 			tokenNames:       map[string]struct{}{},
 			virtualFunctions: map[uint][]*virtualFunction{},
-			freeVFsCount:     len(pff.VirtualFunctions),
+			freeVFsCount:     len(pFun.VirtualFunctions),
 		}
 		p.physicalFunctions[pfPCIAddr] = pf
 
-		for _, serviceDomain := range pff.ServiceDomains {
-			for _, capability := range pff.Capabilities {
+		for _, serviceDomain := range pFun.ServiceDomains {
+			for _, capability := range pFun.Capabilities {
 				pf.tokenNames[path.Join(serviceDomain, capability)] = struct{}{}
 			}
 		}
 
-		for vfPCIAddr, iommuGroup := range pff.VirtualFunctions {
+		for _, vFun := range pFun.VirtualFunctions {
 			vf := &virtualFunction{
-				pciAddr:    vfPCIAddr,
+				pciAddr:    vFun.Address,
 				pfPCIAddr:  pfPCIAddr,
-				iommuGroup: iommuGroup,
+				iommuGroup: vFun.IOMMUGroup,
 			}
-			p.virtualFunctions[vfPCIAddr] = vf
+			p.virtualFunctions[vFun.Address] = vf
 
-			pf.virtualFunctions[iommuGroup] = append(pf.virtualFunctions[iommuGroup], vf)
-			p.iommuGroups[iommuGroup] = sriov.NoDriver
+			pf.virtualFunctions[vFun.IOMMUGroup] = append(pf.virtualFunctions[vFun.IOMMUGroup], vf)
+			p.iommuGroups[vFun.IOMMUGroup] = sriov.NoDriver
 		}
 	}
 
diff --git a/pkg/sriov/token/config.yml b/pkg/sriov/token/config.yml
index f17ccf7a..0832dc97 100644
--- a/pkg/sriov/token/config.yml
+++ b/pkg/sriov/token/config.yml
@@ -7,7 +7,8 @@ physicalFunctions:
     serviceDomains:
       - service.domain.1
     virtualFunctions:
-      0000:01:00.1: 1
+      - address: 0000:01:00.1
+        iommuGroup: 1
   0000:02:00.0:
     capabilities:
       - intel
@@ -16,6 +17,9 @@ physicalFunctions:
       - service.domain.1
       - service.domain.2
     virtualFunctions:
-      0000:02:00.1: 1
-      0000:02:00.2: 2
-      0000:02:00.3: 3
+      - address: 0000:02:00.1
+        iommuGroup: 1
+      - address: 0000:02:00.2
+        iommuGroup: 2
+      - address: 0000:02:00.3
+        iommuGroup: 3
diff --git a/pkg/sriov/token/pool.go b/pkg/sriov/token/pool.go
index 32a060a1..a34af292 100644
--- a/pkg/sriov/token/pool.go
+++ b/pkg/sriov/token/pool.go
@@ -71,11 +71,11 @@ func NewPool(cfg *config.Config) *Pool {
 		closedTokens:  map[string][]*token{},
 	}
 
-	for _, pf := range cfg.PhysicalFunctions {
-		for _, serviceDomain := range pf.ServiceDomains {
-			for _, capability := range pf.Capabilities {
+	for _, pFun := range cfg.PhysicalFunctions {
+		for _, serviceDomain := range pFun.ServiceDomains {
+			for _, capability := range pFun.Capabilities {
 				name := path.Join(serviceDomain, capability)
-				for i := 0; i < len(pf.VirtualFunctions); i++ {
+				for i := 0; i < len(pFun.VirtualFunctions); i++ {
 					tok := &token{
 						id:    uuid.New().String(),
 						name:  name,

From 8949e62cedc7dee053ee7e8e97812f1a81f31e6c Mon Sep 17 00:00:00 2001
From: Vladimir Popov <vladimir.popov@xored.com>
Date: Mon, 26 Oct 2020 11:09:06 +0700
Subject: [PATCH 04/11] Rework resourcepool

Signed-off-by: Vladimir Popov <vladimir.popov@xored.com>
---
 .../common/resourcepool/config.yml            | 24 +++++
 .../common/resourcepool/context.go            | 43 --------
 .../common/resourcepool/init_server.go        | 54 -----------
 .../resourcepool/physical_functions.yml       | 42 ++++----
 .../common/resourcepool/server.go             | 97 +++++++++----------
 .../common/resourcepool/server_test.go        | 47 ++++-----
 pkg/sriov/pci_function.go                     | 13 ---
 pkg/sriov/sriovtest/pci_function.go           |  8 --
 8 files changed, 113 insertions(+), 215 deletions(-)
 create mode 100644 pkg/networkservice/common/resourcepool/config.yml
 delete mode 100644 pkg/networkservice/common/resourcepool/context.go
 delete mode 100644 pkg/networkservice/common/resourcepool/init_server.go

diff --git a/pkg/networkservice/common/resourcepool/config.yml b/pkg/networkservice/common/resourcepool/config.yml
new file mode 100644
index 00000000..6ca0670a
--- /dev/null
+++ b/pkg/networkservice/common/resourcepool/config.yml
@@ -0,0 +1,24 @@
+---
+physicalFunctions:
+  0000:00:01.0:
+    capabilities:
+      - intel
+      - 10G
+    serviceDomains:
+      - service.domain.1
+    virtualFunctions:
+      - address: 0000:00:01.1
+        iommuGroup: 1
+      - address: 0000:00:01.2
+        iommuGroup: 1
+  0000:00:02.0:
+    capabilities:
+      - intel
+      - 10G
+    serviceDomains:
+      - service.domain.1
+    virtualFunctions:
+      - address: 0000:00:02.1
+        iommuGroup: 2
+      - address: 0000:00:02.2
+        iommuGroup: 2
diff --git a/pkg/networkservice/common/resourcepool/context.go b/pkg/networkservice/common/resourcepool/context.go
deleted file mode 100644
index f965cf8a..00000000
--- a/pkg/networkservice/common/resourcepool/context.go
+++ /dev/null
@@ -1,43 +0,0 @@
-// Copyright (c) 2020 Doc.ai and/or its affiliates.
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at:
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package resourcepool
-
-import (
-	"context"
-)
-
-const (
-	resourcePoolKey key = "resourcepool.ResourcePool"
-)
-
-type key string
-
-// WithPool returns a new context with ResourcePool
-func WithPool(parent context.Context, resourcePool ResourcePool) context.Context {
-	if parent == nil {
-		parent = context.TODO()
-	}
-	return context.WithValue(parent, resourcePoolKey, resourcePool)
-}
-
-// Pool returns ResourcePool from context
-func Pool(ctx context.Context) ResourcePool {
-	if rv, ok := ctx.Value(resourcePoolKey).(ResourcePool); ok {
-		return rv
-	}
-	return nil
-}
diff --git a/pkg/networkservice/common/resourcepool/init_server.go b/pkg/networkservice/common/resourcepool/init_server.go
deleted file mode 100644
index 24fa2f1d..00000000
--- a/pkg/networkservice/common/resourcepool/init_server.go
+++ /dev/null
@@ -1,54 +0,0 @@
-// Copyright (c) 2020 Doc.ai and/or its affiliates.
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at:
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package resourcepool
-
-import (
-	"context"
-
-	"github.com/golang/protobuf/ptypes/empty"
-
-	"github.com/networkservicemesh/api/pkg/api/networkservice"
-	"github.com/networkservicemesh/sdk/pkg/networkservice/core/next"
-
-	"github.com/networkservicemesh/sdk-sriov/pkg/sriov/config"
-	"github.com/networkservicemesh/sdk-sriov/pkg/sriov/resource"
-)
-
-type initResourcePoolServer struct {
-	resourcePool *resource.Pool
-}
-
-// NewInitServer returns a new init resource pool server
-func NewInitServer(tokenPool resource.TokenPool, cfg *config.Config) networkservice.NetworkServiceServer {
-	return &initResourcePoolServer{
-		resourcePool: resource.NewPool(tokenPool, cfg),
-	}
-}
-
-func (s *initResourcePoolServer) Request(ctx context.Context, request *networkservice.NetworkServiceRequest) (*networkservice.Connection, error) {
-	if Pool(ctx) == nil {
-		ctx = WithPool(ctx, s.resourcePool)
-	}
-	return next.Server(ctx).Request(ctx, request)
-}
-
-func (s *initResourcePoolServer) Close(ctx context.Context, conn *networkservice.Connection) (*empty.Empty, error) {
-	if Pool(ctx) == nil {
-		ctx = WithPool(ctx, s.resourcePool)
-	}
-	return next.Server(ctx).Close(ctx, conn)
-}
diff --git a/pkg/networkservice/common/resourcepool/physical_functions.yml b/pkg/networkservice/common/resourcepool/physical_functions.yml
index efb1d0df..bf7ca544 100644
--- a/pkg/networkservice/common/resourcepool/physical_functions.yml
+++ b/pkg/networkservice/common/resourcepool/physical_functions.yml
@@ -1,23 +1,29 @@
 ---
-- addr: 0000:00:01.0
+0000:00:01.0:
+  addr: 0000:00:01.0
   ifName: pf-1
+  iommuGroup: 1
+  driver: pf-1-driver
   vfs:
-  - addr: 0000:00:01.1
-    ifName: vf-1-1
-    iommuGroup: 1
-    driver: vf-1-1-driver
-  - addr: 0000:00:01.2
-    ifName: vf-1-2
-    iommuGroup: 1
-    driver: vf-1-2-driver
-- addr: 0000:00:02.0
+    - addr: 0000:00:01.1
+      ifName: vf-1-1
+      iommuGroup: 1
+      driver: vf-1-1-driver
+    - addr: 0000:00:01.2
+      ifName: vf-1-2
+      iommuGroup: 1
+      driver: vf-1-2-driver
+0000:00:02.0:
+  addr: 0000:00:02.0
   ifName: pf-2
+  iommuGroup: 2
+  driver: pf-2-driver
   vfs:
-  - addr: 0000:00:02.1
-    ifName: vf-2-1
-    iommuGroup: 2
-    driver: vf-2-1-driver
-  - addr: 0000:00:02.2
-    ifName: vf-2-2
-    iommuGroup: 2
-    driver: vf-2-2-driver
+    - addr: 0000:00:02.1
+      ifName: vf-2-1
+      iommuGroup: 2
+      driver: vf-2-1-driver
+    - addr: 0000:00:02.2
+      ifName: vf-2-2
+      iommuGroup: 2
+      driver: vf-2-2-driver
diff --git a/pkg/networkservice/common/resourcepool/server.go b/pkg/networkservice/common/resourcepool/server.go
index ba5cc374..ef54e9da 100644
--- a/pkg/networkservice/common/resourcepool/server.go
+++ b/pkg/networkservice/common/resourcepool/server.go
@@ -22,15 +22,16 @@ import (
 	"sync"
 
 	"github.com/golang/protobuf/ptypes/empty"
-	"github.com/networkservicemesh/sdk/pkg/tools/log"
 	"github.com/pkg/errors"
 
 	"github.com/networkservicemesh/api/pkg/api/networkservice"
 	"github.com/networkservicemesh/api/pkg/api/networkservice/mechanisms/vfio"
 	"github.com/networkservicemesh/sdk-kernel/pkg/kernel/networkservice/vfconfig"
 	"github.com/networkservicemesh/sdk/pkg/networkservice/core/next"
+	"github.com/networkservicemesh/sdk/pkg/tools/log"
 
 	"github.com/networkservicemesh/sdk-sriov/pkg/sriov"
+	"github.com/networkservicemesh/sdk-sriov/pkg/sriov/config"
 )
 
 const (
@@ -38,6 +39,12 @@ const (
 	TokenIDKey = "tokenID" // TODO: move to api
 )
 
+// PCIPool is a pci.Pool interface
+type PCIPool interface {
+	GetPCIFunction(pciAddr string) (sriov.PCIFunction, error)
+	BindDriver(iommuGroup uint, driverType sriov.DriverType) error
+}
+
 // ResourcePool is a resource.Pool interface
 type ResourcePool interface {
 	Select(tokenID string, driverType sriov.DriverType) (string, error)
@@ -47,8 +54,9 @@ type ResourcePool interface {
 type resourcePoolServer struct {
 	driverType   sriov.DriverType
 	resourceLock sync.Locker
-	functions    map[sriov.PCIFunction][]sriov.PCIFunction
-	binders      map[uint][]sriov.DriverBinder
+	pciPool      PCIPool
+	resourcePool ResourcePool
+	config       *config.Config
 	selectedVFs  map[string]string
 }
 
@@ -56,14 +64,16 @@ type resourcePoolServer struct {
 func NewServer(
 	driverType sriov.DriverType,
 	resourceLock sync.Locker,
-	functions map[sriov.PCIFunction][]sriov.PCIFunction,
-	binders map[uint][]sriov.DriverBinder,
+	pciPool PCIPool,
+	resourcePool ResourcePool,
+	conf *config.Config,
 ) networkservice.NetworkServiceServer {
 	return &resourcePoolServer{
 		driverType:   driverType,
 		resourceLock: resourceLock,
-		functions:    functions,
-		binders:      binders,
+		pciPool:      pciPool,
+		resourcePool: resourcePool,
+		config:       conf,
 		selectedVFs:  map[string]string{},
 	}
 }
@@ -71,11 +81,6 @@ func NewServer(
 func (s *resourcePoolServer) Request(ctx context.Context, request *networkservice.NetworkServiceRequest) (*networkservice.Connection, error) {
 	logEntry := log.Entry(ctx).WithField("resourcePoolServer", "Request")
 
-	resourcePool := Pool(ctx)
-	if resourcePool == nil {
-		return nil, errors.New("ResourcePool not found")
-	}
-
 	tokenID, ok := request.GetConnection().GetMechanism().GetParameters()[TokenIDKey]
 	if !ok {
 		return nil, errors.New("no token ID provided")
@@ -87,7 +92,7 @@ func (s *resourcePoolServer) Request(ctx context.Context, request *networkservic
 		defer s.resourceLock.Unlock()
 
 		logEntry.Infof("trying to select VF for %v", s.driverType)
-		vf, err := s.selectVF(request.GetConnection().GetId(), vfConfig, resourcePool, tokenID)
+		vf, err := s.selectVF(request.GetConnection().GetId(), vfConfig, tokenID)
 		if err != nil {
 			return err
 		}
@@ -98,7 +103,7 @@ func (s *resourcePoolServer) Request(ctx context.Context, request *networkservic
 			return errors.Wrapf(err, "failed to get VF IOMMU group: %v", vf.GetPCIAddress())
 		}
 
-		if err := s.bindDriver(iommuGroup); err != nil {
+		if err := s.pciPool.BindDriver(iommuGroup, s.driverType); err != nil {
 			return err
 		}
 
@@ -108,76 +113,62 @@ func (s *resourcePoolServer) Request(ctx context.Context, request *networkservic
 
 		return nil
 	}(); err != nil {
-		_ = s.close(ctx, request.GetConnection())
+		_ = s.close(request.GetConnection())
 		return nil, err
 	}
 
 	conn, err := next.Server(ctx).Request(ctx, request)
 	if err != nil {
-		_ = s.close(ctx, request.GetConnection())
+		_ = s.close(request.GetConnection())
 	}
 
 	return conn, err
 }
 
-func (s *resourcePoolServer) selectVF(
-	connID string,
-	vfConfig *vfconfig.VFConfig,
-	resourcePool ResourcePool,
-	tokenID string,
-) (vf sriov.PCIFunction, err error) {
-	s.selectedVFs[connID], err = resourcePool.Select(tokenID, s.driverType)
+func (s *resourcePoolServer) selectVF(connID string, vfConfig *vfconfig.VFConfig, tokenID string) (vf sriov.PCIFunction, err error) {
+	vfPCIAddr, err := s.resourcePool.Select(tokenID, s.driverType)
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to select VF for: %v", s.driverType)
 	}
+	s.selectedVFs[connID] = vfPCIAddr
 
-	for pf, vfs := range s.functions {
-		for i, vf := range vfs {
-			if vf.GetPCIAddress() != s.selectedVFs[connID] {
+	for pfPCIAddr, pFun := range s.config.PhysicalFunctions {
+		for i, vFun := range pFun.VirtualFunctions {
+			if vFun.Address != vfPCIAddr {
 				continue
 			}
 
-			vfConfig.VFInterfaceName, err = vf.GetNetInterfaceName()
+			pf, err := s.pciPool.GetPCIFunction(pfPCIAddr)
 			if err != nil {
-				return nil, errors.Wrapf(err, "failed to get VF net interface name: %v", vf.GetPCIAddress())
+				return nil, errors.Wrapf(err, "failed to get PF: %v", pfPCIAddr)
 			}
 			vfConfig.PFInterfaceName, err = pf.GetNetInterfaceName()
 			if err != nil {
-				return nil, errors.Errorf("failed to get PF net interface name: %v", pf.GetPCIAddress())
+				return nil, errors.Errorf("failed to get PF net interface name: %v", pfPCIAddr)
+			}
+
+			vf, err := s.pciPool.GetPCIFunction(vfPCIAddr)
+			if err != nil {
+				return nil, errors.Wrapf(err, "failed to get VF: %v", vfPCIAddr)
 			}
+			vfConfig.VFInterfaceName, err = vf.GetNetInterfaceName()
+			if err != nil {
+				return nil, errors.Errorf("failed to get VF net interface name: %v", vfPCIAddr)
+			}
+
 			vfConfig.VFNum = i
 
-			return vf, nil
+			return vf, err
 		}
 	}
 
 	return nil, errors.Errorf("no VF with selected PCI address exists: %v", s.selectedVFs[connID])
 }
 
-func (s *resourcePoolServer) bindDriver(igid uint) (err error) {
-	for _, binder := range s.binders[igid] {
-		switch s.driverType {
-		case sriov.KernelDriver:
-			err = binder.BindKernelDriver()
-		case sriov.VFIOPCIDriver:
-			err = binder.BindDriver(string(sriov.VFIOPCIDriver))
-		}
-		if err != nil {
-			return errors.Wrapf(err, "failed to bind driver to IOMMU group: %v", igid)
-		}
-	}
-	return nil
-}
-
 func (s *resourcePoolServer) Close(ctx context.Context, conn *networkservice.Connection) (*empty.Empty, error) {
-	resourcePool := Pool(ctx)
-	if resourcePool == nil {
-		return nil, errors.New("ResourcePool not found")
-	}
-
 	_, err := next.Server(ctx).Close(ctx, conn)
 
-	closeErr := s.close(ctx, conn)
+	closeErr := s.close(conn)
 
 	if err != nil && closeErr != nil {
 		return nil, errors.Wrapf(err, "failed to free VF: %v", closeErr)
@@ -188,7 +179,7 @@ func (s *resourcePoolServer) Close(ctx context.Context, conn *networkservice.Con
 	return &empty.Empty{}, err
 }
 
-func (s *resourcePoolServer) close(ctx context.Context, conn *networkservice.Connection) error {
+func (s *resourcePoolServer) close(conn *networkservice.Connection) error {
 	vfPCIAddr, ok := s.selectedVFs[conn.GetId()]
 	if !ok {
 		return nil
@@ -198,5 +189,5 @@ func (s *resourcePoolServer) close(ctx context.Context, conn *networkservice.Con
 	s.resourceLock.Lock()
 	defer s.resourceLock.Unlock()
 
-	return Pool(ctx).Free(vfPCIAddr)
+	return s.resourcePool.Free(vfPCIAddr)
 }
diff --git a/pkg/networkservice/common/resourcepool/server_test.go b/pkg/networkservice/common/resourcepool/server_test.go
index b3ce9001..0d4476ab 100644
--- a/pkg/networkservice/common/resourcepool/server_test.go
+++ b/pkg/networkservice/common/resourcepool/server_test.go
@@ -27,47 +27,42 @@ import (
 	"github.com/networkservicemesh/api/pkg/api/networkservice"
 	"github.com/networkservicemesh/api/pkg/api/networkservice/mechanisms/vfio"
 	"github.com/networkservicemesh/sdk-kernel/pkg/kernel/networkservice/vfconfig"
-	"github.com/networkservicemesh/sdk/pkg/networkservice/core/chain"
 
 	"github.com/networkservicemesh/sdk-sriov/pkg/networkservice/common/resourcepool"
 	"github.com/networkservicemesh/sdk-sriov/pkg/sriov"
+	"github.com/networkservicemesh/sdk-sriov/pkg/sriov/config"
+	"github.com/networkservicemesh/sdk-sriov/pkg/sriov/pci"
 	"github.com/networkservicemesh/sdk-sriov/pkg/sriov/sriovtest"
 	"github.com/networkservicemesh/sdk-sriov/pkg/tools/yamlhelper"
 )
 
 const (
 	physicalFunctionsFilename = "physical_functions.yml"
+	configFileName            = "config.yml"
+	pf2PciAddr                = "0000:00:02.0"
 )
 
-func initResourcePoolServer(driverType sriov.DriverType) (networkservice.NetworkServiceServer, []*sriovtest.PCIPhysicalFunction) {
-	var pfs []*sriovtest.PCIPhysicalFunction
-	_ = yamlhelper.UnmarshalFile(physicalFunctionsFilename, &pfs)
-
-	functions := map[sriov.PCIFunction][]sriov.PCIFunction{}
-	binders := map[uint][]sriov.DriverBinder{}
-	for _, pf := range pfs {
-		for _, vf := range pf.Vfs {
-			functions[pf] = append(functions[pf], vf)
-			binders[vf.IOMMUGroup] = append(binders[vf.IOMMUGroup], vf)
-		}
-	}
-
-	return chain.NewNetworkServiceServer(resourcepool.NewServer(driverType, &sync.Mutex{}, functions, binders)), pfs
-}
-
 func Test_resourcePoolServer_Request(t *testing.T) {
 	vfConfig := &vfconfig.VFConfig{}
 	ctx := vfconfig.WithConfig(context.TODO(), vfConfig)
 
+	var pfs map[string]*sriovtest.PCIPhysicalFunction
+	_ = yamlhelper.UnmarshalFile(physicalFunctionsFilename, &pfs)
+
+	conf, err := config.ReadConfig(context.TODO(), configFileName)
+	require.NoError(t, err)
+
+	pciPool, err := pci.NewTestPool(pfs, conf)
+	require.NoError(t, err)
+
 	resourcePool := &resourcePoolMock{}
-	ctx = resourcepool.WithPool(ctx, resourcePool)
 
-	server, pfs := initResourcePoolServer(sriov.VFIOPCIDriver)
+	server := resourcepool.NewServer(sriov.VFIOPCIDriver, &sync.Mutex{}, pciPool, resourcePool, conf)
 
 	// 1. Request
 
 	resourcePool.mock.On("Select", "1", sriov.VFIOPCIDriver).
-		Return(pfs[1].Vfs[1].Addr, nil)
+		Return(pfs[pf2PciAddr].Vfs[1].Addr, nil)
 
 	conn, err := server.Request(ctx, &networkservice.NetworkServiceRequest{
 		Connection: &networkservice.Connection{
@@ -84,18 +79,18 @@ func Test_resourcePoolServer_Request(t *testing.T) {
 
 	resourcePool.mock.AssertNumberOfCalls(t, "Select", 1)
 
-	require.Equal(t, pfs[1].Vfs[0].Driver, string(sriov.VFIOPCIDriver))
-	require.Equal(t, pfs[1].Vfs[1].Driver, string(sriov.VFIOPCIDriver))
+	require.Equal(t, pfs[pf2PciAddr].Vfs[0].Driver, string(sriov.VFIOPCIDriver))
+	require.Equal(t, pfs[pf2PciAddr].Vfs[1].Driver, string(sriov.VFIOPCIDriver))
 
-	require.Equal(t, vfConfig.PFInterfaceName, pfs[1].IfName)
-	require.Equal(t, vfConfig.VFInterfaceName, pfs[1].Vfs[1].IfName)
+	require.Equal(t, vfConfig.PFInterfaceName, pfs[pf2PciAddr].IfName)
+	require.Equal(t, vfConfig.VFInterfaceName, pfs[pf2PciAddr].Vfs[1].IfName)
 	require.Equal(t, vfConfig.VFNum, 1)
 
-	require.Equal(t, vfio.ToMechanism(conn.Mechanism).GetIommuGroup(), pfs[1].Vfs[1].IOMMUGroup)
+	require.Equal(t, vfio.ToMechanism(conn.Mechanism).GetIommuGroup(), pfs[pf2PciAddr].Vfs[1].IOMMUGroup)
 
 	// 2. Close
 
-	resourcePool.mock.On("Free", pfs[1].Vfs[1].Addr).
+	resourcePool.mock.On("Free", pfs[pf2PciAddr].Vfs[1].Addr).
 		Return(nil)
 
 	_, err = server.Close(ctx, conn)
diff --git a/pkg/sriov/pci_function.go b/pkg/sriov/pci_function.go
index 40289cfd..018d914e 100644
--- a/pkg/sriov/pci_function.go
+++ b/pkg/sriov/pci_function.go
@@ -17,22 +17,9 @@
 // Package sriov provides types for SR-IOV
 package sriov
 
-// BindablePCIFunction is a PCIFunction and a DriverBinder
-type BindablePCIFunction interface {
-	PCIFunction
-	DriverBinder
-}
-
 // PCIFunction provides methods to get OS PCI function info
 type PCIFunction interface {
 	GetPCIAddress() string
 	GetNetInterfaceName() (string, error)
 	GetIOMMUGroup() (uint, error)
-	GetBoundDriver() (string, error)
-}
-
-// DriverBinder provides a method to bind driver to OS PCI function
-type DriverBinder interface {
-	BindDriver(driver string) error
-	BindKernelDriver() error
 }
diff --git a/pkg/sriov/sriovtest/pci_function.go b/pkg/sriov/sriovtest/pci_function.go
index 806893fd..42209df4 100644
--- a/pkg/sriov/sriovtest/pci_function.go
+++ b/pkg/sriov/sriovtest/pci_function.go
@@ -16,8 +16,6 @@
 
 package sriovtest
 
-import "github.com/networkservicemesh/sdk-sriov/pkg/sriov"
-
 // PCIPhysicalFunction is a test data class for pcifunction.PhysicalFunction
 type PCIPhysicalFunction struct {
 	Vfs []*PCIFunction `yaml:"vfs"`
@@ -58,9 +56,3 @@ func (f *PCIFunction) BindDriver(driver string) error {
 	f.Driver = driver
 	return nil
 }
-
-// BindKernelDriver sets f.Driver = sriov.KernelDriver
-func (f *PCIFunction) BindKernelDriver() error {
-	f.Driver = string(sriov.KernelDriver)
-	return nil
-}

From 85d2d54915e2bca16704561ef10a827e4bdbd2ec Mon Sep 17 00:00:00 2001
From: Vladimir Popov <vladimir.popov@xored.com>
Date: Wed, 28 Oct 2020 12:37:52 +0700
Subject: [PATCH 05/11] Add storage

Signed-off-by: Vladimir Popov <vladimir.popov@xored.com>
---
 pkg/sriov/pci/pool.go             | 35 +++++++++++-----
 pkg/sriov/pci/storage.go          | 31 ++++++++++++++
 pkg/sriov/resource/config.yml     |  1 -
 pkg/sriov/storage/storage.go      | 24 +++++++++++
 pkg/sriov/storage/test_storage.go | 39 ++++++++++++++++++
 pkg/sriov/token/pool.go           | 41 +++++++++++++------
 pkg/sriov/token/pool_test.go      | 18 +++++++-
 pkg/sriov/token/storage.go        | 68 +++++++++++++++++++++++++++++++
 8 files changed, 232 insertions(+), 25 deletions(-)
 create mode 100644 pkg/sriov/pci/storage.go
 create mode 100644 pkg/sriov/storage/storage.go
 create mode 100644 pkg/sriov/storage/test_storage.go
 create mode 100644 pkg/sriov/token/storage.go

diff --git a/pkg/sriov/pci/pool.go b/pkg/sriov/pci/pool.go
index 72148e94..759a51eb 100644
--- a/pkg/sriov/pci/pool.go
+++ b/pkg/sriov/pci/pool.go
@@ -24,6 +24,7 @@ import (
 	"github.com/networkservicemesh/sdk-sriov/pkg/sriov/config"
 	"github.com/networkservicemesh/sdk-sriov/pkg/sriov/pcifunction"
 	"github.com/networkservicemesh/sdk-sriov/pkg/sriov/sriovtest"
+	"github.com/networkservicemesh/sdk-sriov/pkg/sriov/storage"
 )
 
 const (
@@ -49,29 +50,36 @@ type function struct {
 }
 
 // NewPool returns a new PCI Pool
-func NewPool(pciDevicesPath, pciDriversPath string, conf *config.Config) (*Pool, error) {
+func NewPool(pciDevicesPath, pciDriversPath string, store storage.Storage, conf *config.Config) (*Pool, error) {
 	p := &Pool{
 		functions:             map[string]*function{},
 		functionsByIOMMUGroup: map[uint][]*function{},
 	}
 
+	pciStore := &pciStorage{
+		storage: store,
+	}
+
+	kernelDrivers := pciStore.load()
 	for pfPCIAddr := range conf.PhysicalFunctions {
 		pf, err := pcifunction.NewPhysicalFunction(pfPCIAddr, pciDevicesPath, pciDriversPath)
 		if err != nil {
 			return nil, err
 		}
 
-		if err := p.addFunction(&pf.Function); err != nil {
+		if err := p.addFunction(&pf.Function, kernelDrivers); err != nil {
 			return nil, err
 		}
 
 		for _, vf := range pf.GetVirtualFunctions() {
-			if err := p.addFunction(vf); err != nil {
+			if err := p.addFunction(vf, kernelDrivers); err != nil {
 				return nil, err
 			}
 		}
 	}
 
+	pciStore.store(kernelDrivers)
+
 	return p, nil
 }
 
@@ -82,32 +90,39 @@ func NewTestPool(physicalFunctions map[string]*sriovtest.PCIPhysicalFunction, co
 		functionsByIOMMUGroup: map[uint][]*function{},
 	}
 
+	kernelDrivers := map[string]string{}
 	for pfPCIAddr := range conf.PhysicalFunctions {
 		pf, ok := physicalFunctions[pfPCIAddr]
 		if !ok {
 			return nil, errors.Errorf("PF doesn't exist: %v", pfPCIAddr)
 		}
 
-		_ = p.addFunction(&pf.PCIFunction)
+		_ = p.addFunction(&pf.PCIFunction, kernelDrivers)
 
 		for _, vf := range pf.Vfs {
-			_ = p.addFunction(vf)
+			_ = p.addFunction(vf, kernelDrivers)
 		}
 	}
 
 	return p, nil
 }
 
-func (p *Pool) addFunction(pcif pciFunction) (err error) {
+func (p *Pool) addFunction(pcif pciFunction, kernelDrivers map[string]string) (err error) {
 	f := &function{
 		function: pcif,
 	}
 
-	f.kernelDriver, err = pcif.GetBoundDriver()
-	if err != nil {
-		return err
+	pciAddr := pcif.GetPCIAddress()
+
+	var ok bool
+	if f.kernelDriver, ok = kernelDrivers[pciAddr]; !ok {
+		f.kernelDriver, err = pcif.GetBoundDriver()
+		if err != nil {
+			return err
+		}
+		kernelDrivers[pciAddr] = f.kernelDriver
 	}
-	p.functions[pcif.GetPCIAddress()] = f
+	p.functions[pciAddr] = f
 
 	iommuGroup, err := pcif.GetIOMMUGroup()
 	if err != nil {
diff --git a/pkg/sriov/pci/storage.go b/pkg/sriov/pci/storage.go
new file mode 100644
index 00000000..b2e24026
--- /dev/null
+++ b/pkg/sriov/pci/storage.go
@@ -0,0 +1,31 @@
+// Copyright (c) 2020 Doc.ai and/or its affiliates.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pci
+
+import "github.com/networkservicemesh/sdk-sriov/pkg/sriov/storage"
+
+type pciStorage struct {
+	storage storage.Storage
+}
+
+func (s *pciStorage) store(kernelDrivers map[string]string) {
+	s.storage.Store(kernelDrivers)
+}
+
+func (s *pciStorage) load() map[string]string {
+	return s.storage.Load()
+}
diff --git a/pkg/sriov/resource/config.yml b/pkg/sriov/resource/config.yml
index ac391bfe..27e60d6e 100644
--- a/pkg/sriov/resource/config.yml
+++ b/pkg/sriov/resource/config.yml
@@ -33,4 +33,3 @@ physicalFunctions:
         iommuGroup: 1
       - address: 0000:03:00.3
         iommuGroup: 1
-
diff --git a/pkg/sriov/storage/storage.go b/pkg/sriov/storage/storage.go
new file mode 100644
index 00000000..fd4e0124
--- /dev/null
+++ b/pkg/sriov/storage/storage.go
@@ -0,0 +1,24 @@
+// Copyright (c) 2020 Doc.ai and/or its affiliates.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package storage provides interface for key-value storage
+package storage
+
+// Storage is a state storage interface
+type Storage interface {
+	Store(state map[string]string)
+	Load() map[string]string
+}
diff --git a/pkg/sriov/storage/test_storage.go b/pkg/sriov/storage/test_storage.go
new file mode 100644
index 00000000..e59e00d5
--- /dev/null
+++ b/pkg/sriov/storage/test_storage.go
@@ -0,0 +1,39 @@
+// Copyright (c) 2020 Doc.ai and/or its affiliates.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package storage
+
+// TestStorage is a simple map-based Storage for testing
+type TestStorage struct {
+	state map[string]string
+}
+
+// NewTestStorage returns a new TestStorage
+func NewTestStorage() *TestStorage {
+	return &TestStorage{
+		state: map[string]string{},
+	}
+}
+
+// Store stores state
+func (s *TestStorage) Store(state map[string]string) {
+	s.state = state
+}
+
+// Load loads state
+func (s *TestStorage) Load() map[string]string {
+	return s.state
+}
diff --git a/pkg/sriov/token/pool.go b/pkg/sriov/token/pool.go
index a34af292..b5a06739 100644
--- a/pkg/sriov/token/pool.go
+++ b/pkg/sriov/token/pool.go
@@ -25,6 +25,7 @@ import (
 	"github.com/pkg/errors"
 
 	"github.com/networkservicemesh/sdk-sriov/pkg/sriov/config"
+	"github.com/networkservicemesh/sdk-sriov/pkg/sriov/storage"
 )
 
 const (
@@ -64,30 +65,46 @@ type token struct {
 }
 
 // NewPool returns a new Pool
-func NewPool(cfg *config.Config) *Pool {
+func NewPool(store storage.Storage, cfg *config.Config) *Pool {
 	p := &Pool{
 		tokens:        map[string]*token{},
 		tokensByNames: map[string][]*token{},
 		closedTokens:  map[string][]*token{},
 	}
 
-	for _, pFun := range cfg.PhysicalFunctions {
-		for _, serviceDomain := range pFun.ServiceDomains {
-			for _, capability := range pFun.Capabilities {
-				name := path.Join(serviceDomain, capability)
-				for i := 0; i < len(pFun.VirtualFunctions); i++ {
-					tok := &token{
-						id:    uuid.New().String(),
-						name:  name,
-						state: free,
+	tokenStore := &tokenStorage{
+		storage: store,
+	}
+
+	tokens := tokenStore.load()
+	if len(tokens) > 0 {
+		// restore tokens from storage
+		p.tokens = tokens
+		for _, tok := range tokens {
+			p.tokensByNames[tok.name] = append(p.tokensByNames[tok.name], tok)
+		}
+	} else {
+		// create new tokens
+		for _, pFun := range cfg.PhysicalFunctions {
+			for _, serviceDomain := range pFun.ServiceDomains {
+				for _, capability := range pFun.Capabilities {
+					name := path.Join(serviceDomain, capability)
+					for i := 0; i < len(pFun.VirtualFunctions); i++ {
+						tok := &token{
+							id:    uuid.New().String(),
+							name:  name,
+							state: free,
+						}
+						p.tokens[tok.id] = tok
+						p.tokensByNames[tok.name] = append(p.tokensByNames[tok.name], tok)
 					}
-					p.tokens[tok.id] = tok
-					p.tokensByNames[tok.name] = append(p.tokensByNames[tok.name], tok)
 				}
 			}
 		}
 	}
 
+	tokenStore.store(p.tokens)
+
 	return p
 }
 
diff --git a/pkg/sriov/token/pool_test.go b/pkg/sriov/token/pool_test.go
index c8d1d8f0..ea99ae16 100644
--- a/pkg/sriov/token/pool_test.go
+++ b/pkg/sriov/token/pool_test.go
@@ -24,6 +24,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/networkservicemesh/sdk-sriov/pkg/sriov/config"
+	"github.com/networkservicemesh/sdk-sriov/pkg/sriov/storage"
 	"github.com/networkservicemesh/sdk-sriov/pkg/sriov/token"
 )
 
@@ -40,7 +41,7 @@ func TestPool_Tokens(t *testing.T) {
 	cfg, err := config.ReadConfig(context.TODO(), configFileName)
 	require.NoError(t, err)
 
-	p := token.NewPool(cfg)
+	p := token.NewPool(storage.NewTestStorage(), cfg)
 
 	tokens := p.Tokens()
 	require.Equal(t, 5, len(tokens))
@@ -55,7 +56,7 @@ func TestPool_Use(t *testing.T) {
 	cfg, err := config.ReadConfig(context.TODO(), configFileName)
 	require.NoError(t, err)
 
-	p := token.NewPool(cfg)
+	p := token.NewPool(storage.NewTestStorage(), cfg)
 
 	var tokenID string
 	for id := range p.Tokens()[path.Join(serviceDomain2, capability20G)] {
@@ -89,6 +90,19 @@ func TestPool_Use(t *testing.T) {
 	require.Equal(t, 3, countTrue(tokens[path.Join(serviceDomain2, capability20G)]))
 }
 
+func TestPool_Restore(t *testing.T) {
+	cfg, err := config.ReadConfig(context.TODO(), configFileName)
+	require.NoError(t, err)
+
+	store := storage.NewTestStorage()
+
+	p := token.NewPool(store, cfg)
+	tokens := p.Tokens()
+
+	p = token.NewPool(store, cfg)
+	require.Equal(t, tokens, p.Tokens())
+}
+
 func countTrue(m map[string]bool) (count int) {
 	for _, v := range m {
 		if v {
diff --git a/pkg/sriov/token/storage.go b/pkg/sriov/token/storage.go
new file mode 100644
index 00000000..494e6075
--- /dev/null
+++ b/pkg/sriov/token/storage.go
@@ -0,0 +1,68 @@
+// Copyright (c) 2020 Doc.ai and/or its affiliates.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package token
+
+import (
+	"encoding/json"
+
+	"github.com/networkservicemesh/sdk-sriov/pkg/sriov/storage"
+)
+
+type tokenStorage struct {
+	storage storage.Storage
+}
+
+type storedToken struct {
+	ID   string `json:"id"`
+	Name string `json:"name"`
+}
+
+func (s *tokenStorage) store(tokens map[string]*token) {
+	state := map[string]string{}
+	for id, tok := range tokens {
+		state[id] = marshallToken(tok)
+	}
+	s.storage.Store(state)
+}
+
+func marshallToken(tok *token) string {
+	data, _ := json.Marshal(&storedToken{
+		ID:   tok.id,
+		Name: tok.name,
+	})
+	return string(data)
+}
+
+func (s *tokenStorage) load() map[string]*token {
+	tokens := map[string]*token{}
+	state := s.storage.Load()
+	for id, s := range state {
+		tokens[id] = unmarshallToken(s)
+	}
+	return tokens
+}
+
+func unmarshallToken(s string) (tok *token) {
+	storedTok := storedToken{}
+	_ = json.Unmarshal([]byte(s), &storedTok)
+
+	return &token{
+		id:    storedTok.ID,
+		name:  storedTok.Name,
+		state: free,
+	}
+}

From 3501bbb73d8a4bffd28506bc9d948c6387a378af Mon Sep 17 00:00:00 2001
From: Vladimir Popov <vladimir.popov@xored.com>
Date: Wed, 9 Dec 2020 17:25:37 +0700
Subject: [PATCH 06/11] Rework config

Signed-off-by: Vladimir Popov <vladimir.popov@xored.com>
---
 .../common/resourcepool/config.yml            |  4 +++
 .../resourcepool/physical_functions.yml       |  8 ++---
 .../common/resourcepool/server.go             | 10 +++---
 pkg/sriov/config/config.go                    | 34 ++++++++++++++++---
 pkg/sriov/config/config.yml                   |  4 +++
 pkg/sriov/config/config_test.go               |  6 ++++
 pkg/sriov/pci/pool.go                         |  8 ++---
 pkg/sriov/pci/update_config.go                |  6 ++--
 pkg/sriov/resource/config.yml                 |  6 ++++
 pkg/sriov/token/config.yml                    |  4 +++
 pkg/sriov/token/pool.go                       |  8 ++---
 11 files changed, 73 insertions(+), 25 deletions(-)

diff --git a/pkg/networkservice/common/resourcepool/config.yml b/pkg/networkservice/common/resourcepool/config.yml
index 6ca0670a..4da02cfb 100644
--- a/pkg/networkservice/common/resourcepool/config.yml
+++ b/pkg/networkservice/common/resourcepool/config.yml
@@ -1,6 +1,8 @@
 ---
 physicalFunctions:
   0000:00:01.0:
+    pfKernelDriver: pf-1-driver
+    vfKernelDriver: vf-1-driver
     capabilities:
       - intel
       - 10G
@@ -12,6 +14,8 @@ physicalFunctions:
       - address: 0000:00:01.2
         iommuGroup: 1
   0000:00:02.0:
+    pfKernelDriver: pf-2-driver
+    vfKernelDriver: vf-2-driver
     capabilities:
       - intel
       - 10G
diff --git a/pkg/networkservice/common/resourcepool/physical_functions.yml b/pkg/networkservice/common/resourcepool/physical_functions.yml
index bf7ca544..15f504a2 100644
--- a/pkg/networkservice/common/resourcepool/physical_functions.yml
+++ b/pkg/networkservice/common/resourcepool/physical_functions.yml
@@ -8,11 +8,11 @@
     - addr: 0000:00:01.1
       ifName: vf-1-1
       iommuGroup: 1
-      driver: vf-1-1-driver
+      driver: vf-1-driver
     - addr: 0000:00:01.2
       ifName: vf-1-2
       iommuGroup: 1
-      driver: vf-1-2-driver
+      driver: vf-1-driver
 0000:00:02.0:
   addr: 0000:00:02.0
   ifName: pf-2
@@ -22,8 +22,8 @@
     - addr: 0000:00:02.1
       ifName: vf-2-1
       iommuGroup: 2
-      driver: vf-2-1-driver
+      driver: vf-2-driver
     - addr: 0000:00:02.2
       ifName: vf-2-2
       iommuGroup: 2
-      driver: vf-2-2-driver
+      driver: vf-2-driver
diff --git a/pkg/networkservice/common/resourcepool/server.go b/pkg/networkservice/common/resourcepool/server.go
index ef54e9da..92d7c5b7 100644
--- a/pkg/networkservice/common/resourcepool/server.go
+++ b/pkg/networkservice/common/resourcepool/server.go
@@ -66,14 +66,14 @@ func NewServer(
 	resourceLock sync.Locker,
 	pciPool PCIPool,
 	resourcePool ResourcePool,
-	conf *config.Config,
+	cfg *config.Config,
 ) networkservice.NetworkServiceServer {
 	return &resourcePoolServer{
 		driverType:   driverType,
 		resourceLock: resourceLock,
 		pciPool:      pciPool,
 		resourcePool: resourcePool,
-		config:       conf,
+		config:       cfg,
 		selectedVFs:  map[string]string{},
 	}
 }
@@ -132,9 +132,9 @@ func (s *resourcePoolServer) selectVF(connID string, vfConfig *vfconfig.VFConfig
 	}
 	s.selectedVFs[connID] = vfPCIAddr
 
-	for pfPCIAddr, pFun := range s.config.PhysicalFunctions {
-		for i, vFun := range pFun.VirtualFunctions {
-			if vFun.Address != vfPCIAddr {
+	for pfPCIAddr, pfCfg := range s.config.PhysicalFunctions {
+		for i, vfCfg := range pfCfg.VirtualFunctions {
+			if vfCfg.Address != vfPCIAddr {
 				continue
 			}
 
diff --git a/pkg/sriov/config/config.go b/pkg/sriov/config/config.go
index 833fb773..44c68c82 100644
--- a/pkg/sriov/config/config.go
+++ b/pkg/sriov/config/config.go
@@ -23,6 +23,7 @@ import (
 	"strings"
 
 	"github.com/networkservicemesh/sdk/pkg/tools/log"
+	"github.com/pkg/errors"
 
 	"github.com/networkservicemesh/sdk-sriov/pkg/tools/yamlhelper"
 )
@@ -50,6 +51,8 @@ func (c *Config) String() string {
 
 // PhysicalFunction contains physical function capabilities, available services domains and virtual functions
 type PhysicalFunction struct {
+	PFKernelDriver   string             `yaml:"pfKernelDriver"`
+	VFKernelDriver   string             `yaml:"vfKernelDriver"`
 	Capabilities     []string           `yaml:"capabilities"`
 	ServiceDomains   []string           `yaml:"serviceDomains"`
 	VirtualFunctions []*VirtualFunction `yaml:"virtualFunctions"`
@@ -59,7 +62,13 @@ func (pf *PhysicalFunction) String() string {
 	sb := &strings.Builder{}
 	_, _ = sb.WriteString("&{")
 
-	_, _ = sb.WriteString("Capabilities:[")
+	_, _ = sb.WriteString("PFKernelDriver:")
+	_, _ = sb.WriteString(pf.PFKernelDriver)
+
+	_, _ = sb.WriteString(" VFKernelDriver:")
+	_, _ = sb.WriteString(pf.VFKernelDriver)
+
+	_, _ = sb.WriteString(" Capabilities:[")
 	_, _ = sb.WriteString(strings.Join(pf.Capabilities, " "))
 	_, _ = sb.WriteString("]")
 
@@ -89,12 +98,27 @@ type VirtualFunction struct {
 func ReadConfig(ctx context.Context, configFile string) (*Config, error) {
 	logEntry := log.Entry(ctx).WithField("Config", "ReadConfig")
 
-	config := &Config{}
-	if err := yamlhelper.UnmarshalFile(configFile, config); err != nil {
+	cfg := &Config{}
+	if err := yamlhelper.UnmarshalFile(configFile, cfg); err != nil {
 		return nil, err
 	}
 
-	logEntry.Infof("unmarshalled Config: %+v", config)
+	for pciAddr, pfCfg := range cfg.PhysicalFunctions {
+		if pfCfg.PFKernelDriver == "" {
+			return nil, errors.Errorf("%s has no PFKernelDriver set", pciAddr)
+		}
+		if pfCfg.VFKernelDriver == "" {
+			return nil, errors.Errorf("%s has no VFKernelDriver set", pciAddr)
+		}
+		if len(pfCfg.Capabilities) == 0 {
+			return nil, errors.Errorf("%s has no Capabilities set", pciAddr)
+		}
+		if len(pfCfg.ServiceDomains) == 0 {
+			return nil, errors.Errorf("%s has no ServiceDomains set", pciAddr)
+		}
+	}
+
+	logEntry.Infof("unmarshalled Config: %+v", cfg)
 
-	return config, nil
+	return cfg, nil
 }
diff --git a/pkg/sriov/config/config.yml b/pkg/sriov/config/config.yml
index 2a3c0384..31352d38 100644
--- a/pkg/sriov/config/config.yml
+++ b/pkg/sriov/config/config.yml
@@ -1,6 +1,8 @@
 ---
 physicalFunctions:
   0000:01:00.0:
+    pfKernelDriver: pf-driver
+    vfKernelDriver: vf-driver
     capabilities:
       - intel
       - 10G
@@ -12,6 +14,8 @@ physicalFunctions:
       - address: 0000:01:00.2
         iommuGroup: 2
   0000:02:00.0:
+    pfKernelDriver: pf-driver
+    vfKernelDriver: vf-driver
     capabilities:
       - intel
       - 20G
diff --git a/pkg/sriov/config/config_test.go b/pkg/sriov/config/config_test.go
index b8d78b34..e2344eb1 100644
--- a/pkg/sriov/config/config_test.go
+++ b/pkg/sriov/config/config_test.go
@@ -29,6 +29,8 @@ const (
 	configFileName  = "config.yml"
 	pf1PciAddr      = "0000:01:00.0"
 	pf2PciAddr      = "0000:02:00.0"
+	pfKernelDriver  = "pf-driver"
+	vfKernelDriver  = "vf-driver"
 	capabilityIntel = "intel"
 	capability10G   = "10G"
 	capability20G   = "20G"
@@ -47,6 +49,8 @@ func TestReadConfigFile(t *testing.T) {
 	require.Equal(t, &config.Config{
 		PhysicalFunctions: map[string]*config.PhysicalFunction{
 			pf1PciAddr: {
+				PFKernelDriver: pfKernelDriver,
+				VFKernelDriver: vfKernelDriver,
 				Capabilities: []string{
 					capabilityIntel,
 					capability10G,
@@ -66,6 +70,8 @@ func TestReadConfigFile(t *testing.T) {
 				},
 			},
 			pf2PciAddr: {
+				PFKernelDriver: pfKernelDriver,
+				VFKernelDriver: vfKernelDriver,
 				Capabilities: []string{
 					capabilityIntel,
 					capability20G,
diff --git a/pkg/sriov/pci/pool.go b/pkg/sriov/pci/pool.go
index 759a51eb..895e0a36 100644
--- a/pkg/sriov/pci/pool.go
+++ b/pkg/sriov/pci/pool.go
@@ -50,7 +50,7 @@ type function struct {
 }
 
 // NewPool returns a new PCI Pool
-func NewPool(pciDevicesPath, pciDriversPath string, store storage.Storage, conf *config.Config) (*Pool, error) {
+func NewPool(pciDevicesPath, pciDriversPath string, store storage.Storage, cfg *config.Config) (*Pool, error) {
 	p := &Pool{
 		functions:             map[string]*function{},
 		functionsByIOMMUGroup: map[uint][]*function{},
@@ -61,7 +61,7 @@ func NewPool(pciDevicesPath, pciDriversPath string, store storage.Storage, conf
 	}
 
 	kernelDrivers := pciStore.load()
-	for pfPCIAddr := range conf.PhysicalFunctions {
+	for pfPCIAddr := range cfg.PhysicalFunctions {
 		pf, err := pcifunction.NewPhysicalFunction(pfPCIAddr, pciDevicesPath, pciDriversPath)
 		if err != nil {
 			return nil, err
@@ -84,14 +84,14 @@ func NewPool(pciDevicesPath, pciDriversPath string, store storage.Storage, conf
 }
 
 // NewTestPool returns a new PCI Pool for testing
-func NewTestPool(physicalFunctions map[string]*sriovtest.PCIPhysicalFunction, conf *config.Config) (*Pool, error) {
+func NewTestPool(physicalFunctions map[string]*sriovtest.PCIPhysicalFunction, cfg *config.Config) (*Pool, error) {
 	p := &Pool{
 		functions:             map[string]*function{},
 		functionsByIOMMUGroup: map[uint][]*function{},
 	}
 
 	kernelDrivers := map[string]string{}
-	for pfPCIAddr := range conf.PhysicalFunctions {
+	for pfPCIAddr := range cfg.PhysicalFunctions {
 		pf, ok := physicalFunctions[pfPCIAddr]
 		if !ok {
 			return nil, errors.Errorf("PF doesn't exist: %v", pfPCIAddr)
diff --git a/pkg/sriov/pci/update_config.go b/pkg/sriov/pci/update_config.go
index 96238d0c..00ae5a35 100644
--- a/pkg/sriov/pci/update_config.go
+++ b/pkg/sriov/pci/update_config.go
@@ -22,8 +22,8 @@ import (
 )
 
 // UpdateConfig updates config with virtual functions
-func UpdateConfig(pciDevicesPath, pciDriversPath string, conf *config.Config) error {
-	for pfPCIAddr, pFun := range conf.PhysicalFunctions {
+func UpdateConfig(pciDevicesPath, pciDriversPath string, cfg *config.Config) error {
+	for pfPCIAddr, pfCfg := range cfg.PhysicalFunctions {
 		pf, err := pcifunction.NewPhysicalFunction(pfPCIAddr, pciDevicesPath, pciDriversPath)
 		if err != nil {
 			return err
@@ -35,7 +35,7 @@ func UpdateConfig(pciDevicesPath, pciDriversPath string, conf *config.Config) er
 				return err
 			}
 
-			pFun.VirtualFunctions = append(pFun.VirtualFunctions, &config.VirtualFunction{
+			pfCfg.VirtualFunctions = append(pfCfg.VirtualFunctions, &config.VirtualFunction{
 				Address:    vf.GetPCIAddress(),
 				IOMMUGroup: iommuGroup,
 			})
diff --git a/pkg/sriov/resource/config.yml b/pkg/sriov/resource/config.yml
index 27e60d6e..ae1b9a5e 100644
--- a/pkg/sriov/resource/config.yml
+++ b/pkg/sriov/resource/config.yml
@@ -1,6 +1,8 @@
 ---
 physicalFunctions:
   0000:01:00.0:
+    pfKernelDriver: pf-driver
+    vfKernelDriver: vf-driver
     capabilities:
       - intel
       - 10G
@@ -10,6 +12,8 @@ physicalFunctions:
       - address: 0000:01:00.1
         iommuGroup: 1
   0000:02:00.0:
+    pfKernelDriver: pf-driver
+    vfKernelDriver: vf-driver
     capabilities:
       - intel
       - 10G
@@ -21,6 +25,8 @@ physicalFunctions:
       - address: 0000:02:00.2
         iommuGroup: 2
   0000:03:00.0:
+    pfKernelDriver: pf-driver
+    vfKernelDriver: vf-driver
     capabilities:
       - intel
       - 20G
diff --git a/pkg/sriov/token/config.yml b/pkg/sriov/token/config.yml
index 0832dc97..12265efc 100644
--- a/pkg/sriov/token/config.yml
+++ b/pkg/sriov/token/config.yml
@@ -1,6 +1,8 @@
 ---
 physicalFunctions:
   0000:01:00.0:
+    pfKernelDriver: pf-driver
+    vfKernelDriver: vf-driver
     capabilities:
       - intel
       - 10G
@@ -10,6 +12,8 @@ physicalFunctions:
       - address: 0000:01:00.1
         iommuGroup: 1
   0000:02:00.0:
+    pfKernelDriver: pf-driver
+    vfKernelDriver: vf-driver
     capabilities:
       - intel
       - 20G
diff --git a/pkg/sriov/token/pool.go b/pkg/sriov/token/pool.go
index b5a06739..71b390ca 100644
--- a/pkg/sriov/token/pool.go
+++ b/pkg/sriov/token/pool.go
@@ -85,11 +85,11 @@ func NewPool(store storage.Storage, cfg *config.Config) *Pool {
 		}
 	} else {
 		// create new tokens
-		for _, pFun := range cfg.PhysicalFunctions {
-			for _, serviceDomain := range pFun.ServiceDomains {
-				for _, capability := range pFun.Capabilities {
+		for _, pfCfg := range cfg.PhysicalFunctions {
+			for _, serviceDomain := range pfCfg.ServiceDomains {
+				for _, capability := range pfCfg.Capabilities {
 					name := path.Join(serviceDomain, capability)
-					for i := 0; i < len(pFun.VirtualFunctions); i++ {
+					for i := 0; i < len(pfCfg.VirtualFunctions); i++ {
 						tok := &token{
 							id:    uuid.New().String(),
 							name:  name,

From 536aa2dfb1484a5e416d1cf8123e993912e95e11 Mon Sep 17 00:00:00 2001
From: Vladimir Popov <vladimir.popov@xored.com>
Date: Wed, 9 Dec 2020 17:29:30 +0700
Subject: [PATCH 07/11] Rework PCIPool without storage

Signed-off-by: Vladimir Popov <vladimir.popov@xored.com>
---
 pkg/sriov/pci/pool.go    | 40 +++++++++++-----------------------------
 pkg/sriov/pci/storage.go | 31 -------------------------------
 2 files changed, 11 insertions(+), 60 deletions(-)
 delete mode 100644 pkg/sriov/pci/storage.go

diff --git a/pkg/sriov/pci/pool.go b/pkg/sriov/pci/pool.go
index 895e0a36..1da619d8 100644
--- a/pkg/sriov/pci/pool.go
+++ b/pkg/sriov/pci/pool.go
@@ -24,7 +24,6 @@ import (
 	"github.com/networkservicemesh/sdk-sriov/pkg/sriov/config"
 	"github.com/networkservicemesh/sdk-sriov/pkg/sriov/pcifunction"
 	"github.com/networkservicemesh/sdk-sriov/pkg/sriov/sriovtest"
-	"github.com/networkservicemesh/sdk-sriov/pkg/sriov/storage"
 )
 
 const (
@@ -50,36 +49,29 @@ type function struct {
 }
 
 // NewPool returns a new PCI Pool
-func NewPool(pciDevicesPath, pciDriversPath string, store storage.Storage, cfg *config.Config) (*Pool, error) {
+func NewPool(pciDevicesPath, pciDriversPath string, cfg *config.Config) (*Pool, error) {
 	p := &Pool{
 		functions:             map[string]*function{},
 		functionsByIOMMUGroup: map[uint][]*function{},
 	}
 
-	pciStore := &pciStorage{
-		storage: store,
-	}
-
-	kernelDrivers := pciStore.load()
-	for pfPCIAddr := range cfg.PhysicalFunctions {
+	for pfPCIAddr, pfCfg := range cfg.PhysicalFunctions {
 		pf, err := pcifunction.NewPhysicalFunction(pfPCIAddr, pciDevicesPath, pciDriversPath)
 		if err != nil {
 			return nil, err
 		}
 
-		if err := p.addFunction(&pf.Function, kernelDrivers); err != nil {
+		if err := p.addFunction(&pf.Function, pfCfg.PFKernelDriver); err != nil {
 			return nil, err
 		}
 
 		for _, vf := range pf.GetVirtualFunctions() {
-			if err := p.addFunction(vf, kernelDrivers); err != nil {
+			if err := p.addFunction(vf, pfCfg.VFKernelDriver); err != nil {
 				return nil, err
 			}
 		}
 	}
 
-	pciStore.store(kernelDrivers)
-
 	return p, nil
 }
 
@@ -90,39 +82,29 @@ func NewTestPool(physicalFunctions map[string]*sriovtest.PCIPhysicalFunction, cf
 		functionsByIOMMUGroup: map[uint][]*function{},
 	}
 
-	kernelDrivers := map[string]string{}
-	for pfPCIAddr := range cfg.PhysicalFunctions {
+	for pfPCIAddr, pfCfg := range cfg.PhysicalFunctions {
 		pf, ok := physicalFunctions[pfPCIAddr]
 		if !ok {
 			return nil, errors.Errorf("PF doesn't exist: %v", pfPCIAddr)
 		}
 
-		_ = p.addFunction(&pf.PCIFunction, kernelDrivers)
+		_ = p.addFunction(&pf.PCIFunction, pfCfg.PFKernelDriver)
 
 		for _, vf := range pf.Vfs {
-			_ = p.addFunction(vf, kernelDrivers)
+			_ = p.addFunction(vf, pfCfg.VFKernelDriver)
 		}
 	}
 
 	return p, nil
 }
 
-func (p *Pool) addFunction(pcif pciFunction, kernelDrivers map[string]string) (err error) {
+func (p *Pool) addFunction(pcif pciFunction, kernelDriver string) (err error) {
 	f := &function{
-		function: pcif,
+		function:     pcif,
+		kernelDriver: kernelDriver,
 	}
 
-	pciAddr := pcif.GetPCIAddress()
-
-	var ok bool
-	if f.kernelDriver, ok = kernelDrivers[pciAddr]; !ok {
-		f.kernelDriver, err = pcif.GetBoundDriver()
-		if err != nil {
-			return err
-		}
-		kernelDrivers[pciAddr] = f.kernelDriver
-	}
-	p.functions[pciAddr] = f
+	p.functions[pcif.GetPCIAddress()] = f
 
 	iommuGroup, err := pcif.GetIOMMUGroup()
 	if err != nil {
diff --git a/pkg/sriov/pci/storage.go b/pkg/sriov/pci/storage.go
deleted file mode 100644
index b2e24026..00000000
--- a/pkg/sriov/pci/storage.go
+++ /dev/null
@@ -1,31 +0,0 @@
-// Copyright (c) 2020 Doc.ai and/or its affiliates.
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at:
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package pci
-
-import "github.com/networkservicemesh/sdk-sriov/pkg/sriov/storage"
-
-type pciStorage struct {
-	storage storage.Storage
-}
-
-func (s *pciStorage) store(kernelDrivers map[string]string) {
-	s.storage.Store(kernelDrivers)
-}
-
-func (s *pciStorage) load() map[string]string {
-	return s.storage.Load()
-}

From e8eaca1fe0736ae6757f40ac40fd132d48de51f0 Mon Sep 17 00:00:00 2001
From: Vladimir Popov <vladimir.popov@xored.com>
Date: Wed, 9 Dec 2020 17:41:59 +0700
Subject: [PATCH 08/11] Rework token pool without storage

Signed-off-by: Vladimir Popov <vladimir.popov@xored.com>
---
 pkg/sriov/storage/storage.go      | 24 ---------
 pkg/sriov/storage/test_storage.go | 39 --------------
 pkg/sriov/token/pool.go           | 87 ++++++++++++++++++++-----------
 pkg/sriov/token/pool_test.go      | 19 ++++---
 pkg/sriov/token/storage.go        | 68 ------------------------
 5 files changed, 69 insertions(+), 168 deletions(-)
 delete mode 100644 pkg/sriov/storage/storage.go
 delete mode 100644 pkg/sriov/storage/test_storage.go
 delete mode 100644 pkg/sriov/token/storage.go

diff --git a/pkg/sriov/storage/storage.go b/pkg/sriov/storage/storage.go
deleted file mode 100644
index fd4e0124..00000000
--- a/pkg/sriov/storage/storage.go
+++ /dev/null
@@ -1,24 +0,0 @@
-// Copyright (c) 2020 Doc.ai and/or its affiliates.
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at:
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// Package storage provides interface for key-value storage
-package storage
-
-// Storage is a state storage interface
-type Storage interface {
-	Store(state map[string]string)
-	Load() map[string]string
-}
diff --git a/pkg/sriov/storage/test_storage.go b/pkg/sriov/storage/test_storage.go
deleted file mode 100644
index e59e00d5..00000000
--- a/pkg/sriov/storage/test_storage.go
+++ /dev/null
@@ -1,39 +0,0 @@
-// Copyright (c) 2020 Doc.ai and/or its affiliates.
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at:
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package storage
-
-// TestStorage is a simple map-based Storage for testing
-type TestStorage struct {
-	state map[string]string
-}
-
-// NewTestStorage returns a new TestStorage
-func NewTestStorage() *TestStorage {
-	return &TestStorage{
-		state: map[string]string{},
-	}
-}
-
-// Store stores state
-func (s *TestStorage) Store(state map[string]string) {
-	s.state = state
-}
-
-// Load loads state
-func (s *TestStorage) Load() map[string]string {
-	return s.state
-}
diff --git a/pkg/sriov/token/pool.go b/pkg/sriov/token/pool.go
index 71b390ca..35f2188e 100644
--- a/pkg/sriov/token/pool.go
+++ b/pkg/sriov/token/pool.go
@@ -25,7 +25,6 @@ import (
 	"github.com/pkg/errors"
 
 	"github.com/networkservicemesh/sdk-sriov/pkg/sriov/config"
-	"github.com/networkservicemesh/sdk-sriov/pkg/sriov/storage"
 )
 
 const (
@@ -37,11 +36,12 @@ const (
 
 // Pool manages forwarder SR-IOV resource tokens
 type Pool struct {
-	lock          sync.Mutex
 	tokens        map[string]*token   // tokens[id] -> *token
 	tokensByNames map[string][]*token // tokensByNames[name] -> []*token
 	closedTokens  map[string][]*token // closedTokens[id] -> []*token
 	listeners     []func()
+	lock          sync.Mutex
+	dirty         bool
 }
 
 type state int
@@ -65,49 +65,64 @@ type token struct {
 }
 
 // NewPool returns a new Pool
-func NewPool(store storage.Storage, cfg *config.Config) *Pool {
+func NewPool(cfg *config.Config) *Pool {
 	p := &Pool{
 		tokens:        map[string]*token{},
 		tokensByNames: map[string][]*token{},
 		closedTokens:  map[string][]*token{},
 	}
 
-	tokenStore := &tokenStorage{
-		storage: store,
-	}
-
-	tokens := tokenStore.load()
-	if len(tokens) > 0 {
-		// restore tokens from storage
-		p.tokens = tokens
-		for _, tok := range tokens {
-			p.tokensByNames[tok.name] = append(p.tokensByNames[tok.name], tok)
-		}
-	} else {
-		// create new tokens
-		for _, pfCfg := range cfg.PhysicalFunctions {
-			for _, serviceDomain := range pfCfg.ServiceDomains {
-				for _, capability := range pfCfg.Capabilities {
-					name := path.Join(serviceDomain, capability)
-					for i := 0; i < len(pfCfg.VirtualFunctions); i++ {
-						tok := &token{
-							id:    uuid.New().String(),
-							name:  name,
-							state: free,
-						}
-						p.tokens[tok.id] = tok
-						p.tokensByNames[tok.name] = append(p.tokensByNames[tok.name], tok)
+	for _, pfCfg := range cfg.PhysicalFunctions {
+		for _, serviceDomain := range pfCfg.ServiceDomains {
+			for _, capability := range pfCfg.Capabilities {
+				name := path.Join(serviceDomain, capability)
+				for i := 0; i < len(pfCfg.VirtualFunctions); i++ {
+					tok := &token{
+						id:    uuid.New().String(),
+						name:  name,
+						state: free,
 					}
+					p.tokens[tok.id] = tok
+					p.tokensByNames[tok.name] = append(p.tokensByNames[tok.name], tok)
 				}
 			}
 		}
 	}
 
-	tokenStore.store(p.tokens)
-
 	return p
 }
 
+// Restore replaces part of existing tokens with given tokens and set them into the allocated state
+// NOTE: it can be called only on untouched Pool, any actions will disable Restore
+func (p *Pool) Restore(tokens map[string][]string) error {
+	p.lock.Lock()
+	defer p.lock.Unlock()
+
+	if p.dirty {
+		return errors.New("token pool has already been accessed")
+	}
+	p.dirty = true
+
+	for name, ids := range tokens {
+		toks, ok := p.tokensByNames[name]
+		if !ok {
+			continue
+		}
+
+		for i := 0; i < len(ids) && i < len(toks); i++ {
+			tok := toks[i]
+			delete(p.tokens, tok.id)
+
+			tok.id = ids[i]
+			tok.state = allocated
+
+			p.tokens[tok.id] = tok
+		}
+	}
+
+	return nil
+}
+
 // AddListener adds a new listener that fires on tokens state change to/from "closed"
 func (p *Pool) AddListener(listener func()) {
 	p.lock.Lock()
@@ -121,6 +136,8 @@ func (p *Pool) Tokens() map[string]map[string]bool {
 	p.lock.Lock()
 	defer p.lock.Unlock()
 
+	p.dirty = true
+
 	tokens := map[string]map[string]bool{}
 	for name, toks := range p.tokensByNames {
 		tokens[name] = map[string]bool{}
@@ -136,6 +153,8 @@ func (p *Pool) Find(id string) (string, error) {
 	p.lock.Lock()
 	defer p.lock.Unlock()
 
+	p.dirty = true
+
 	tok, err := p.find(id)
 	if err != nil {
 		return "", err
@@ -159,6 +178,8 @@ func (p *Pool) Allocate(id string) error {
 	p.lock.Lock()
 	defer p.lock.Unlock()
 
+	p.dirty = true
+
 	tok, err := p.find(id)
 	if err != nil {
 		return err
@@ -184,6 +205,8 @@ func (p *Pool) Free(id string) error {
 	p.lock.Lock()
 	defer p.lock.Unlock()
 
+	p.dirty = true
+
 	tok, err := p.find(id)
 	if err != nil {
 		return err
@@ -209,6 +232,8 @@ func (p *Pool) Use(id string, names []string) error {
 	p.lock.Lock()
 	defer p.lock.Unlock()
 
+	p.dirty = true
+
 	tok, err := p.find(id)
 	if err != nil {
 		return err
@@ -263,6 +288,8 @@ func (p *Pool) StopUsing(id string) error {
 	p.lock.Lock()
 	defer p.lock.Unlock()
 
+	p.dirty = true
+
 	return p.stopUsing(id)
 }
 
diff --git a/pkg/sriov/token/pool_test.go b/pkg/sriov/token/pool_test.go
index ea99ae16..1b6eb1f2 100644
--- a/pkg/sriov/token/pool_test.go
+++ b/pkg/sriov/token/pool_test.go
@@ -24,7 +24,6 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/networkservicemesh/sdk-sriov/pkg/sriov/config"
-	"github.com/networkservicemesh/sdk-sriov/pkg/sriov/storage"
 	"github.com/networkservicemesh/sdk-sriov/pkg/sriov/token"
 )
 
@@ -41,7 +40,7 @@ func TestPool_Tokens(t *testing.T) {
 	cfg, err := config.ReadConfig(context.TODO(), configFileName)
 	require.NoError(t, err)
 
-	p := token.NewPool(storage.NewTestStorage(), cfg)
+	p := token.NewPool(cfg)
 
 	tokens := p.Tokens()
 	require.Equal(t, 5, len(tokens))
@@ -56,7 +55,7 @@ func TestPool_Use(t *testing.T) {
 	cfg, err := config.ReadConfig(context.TODO(), configFileName)
 	require.NoError(t, err)
 
-	p := token.NewPool(storage.NewTestStorage(), cfg)
+	p := token.NewPool(cfg)
 
 	var tokenID string
 	for id := range p.Tokens()[path.Join(serviceDomain2, capability20G)] {
@@ -94,12 +93,18 @@ func TestPool_Restore(t *testing.T) {
 	cfg, err := config.ReadConfig(context.TODO(), configFileName)
 	require.NoError(t, err)
 
-	store := storage.NewTestStorage()
-
-	p := token.NewPool(store, cfg)
+	p := token.NewPool(cfg)
 	tokens := p.Tokens()
 
-	p = token.NewPool(store, cfg)
+	idsByNames := map[string][]string{}
+	for name, toks := range tokens {
+		for id := range toks {
+			idsByNames[name] = append(idsByNames[name], id)
+		}
+	}
+
+	p = token.NewPool(cfg)
+	require.NoError(t, p.Restore(idsByNames))
 	require.Equal(t, tokens, p.Tokens())
 }
 
diff --git a/pkg/sriov/token/storage.go b/pkg/sriov/token/storage.go
deleted file mode 100644
index 494e6075..00000000
--- a/pkg/sriov/token/storage.go
+++ /dev/null
@@ -1,68 +0,0 @@
-// Copyright (c) 2020 Doc.ai and/or its affiliates.
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at:
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package token
-
-import (
-	"encoding/json"
-
-	"github.com/networkservicemesh/sdk-sriov/pkg/sriov/storage"
-)
-
-type tokenStorage struct {
-	storage storage.Storage
-}
-
-type storedToken struct {
-	ID   string `json:"id"`
-	Name string `json:"name"`
-}
-
-func (s *tokenStorage) store(tokens map[string]*token) {
-	state := map[string]string{}
-	for id, tok := range tokens {
-		state[id] = marshallToken(tok)
-	}
-	s.storage.Store(state)
-}
-
-func marshallToken(tok *token) string {
-	data, _ := json.Marshal(&storedToken{
-		ID:   tok.id,
-		Name: tok.name,
-	})
-	return string(data)
-}
-
-func (s *tokenStorage) load() map[string]*token {
-	tokens := map[string]*token{}
-	state := s.storage.Load()
-	for id, s := range state {
-		tokens[id] = unmarshallToken(s)
-	}
-	return tokens
-}
-
-func unmarshallToken(s string) (tok *token) {
-	storedTok := storedToken{}
-	_ = json.Unmarshal([]byte(s), &storedTok)
-
-	return &token{
-		id:    storedTok.ID,
-		name:  storedTok.Name,
-		state: free,
-	}
-}

From 5da49d793eb5ab11c4608562d857b686d6200d5d Mon Sep 17 00:00:00 2001
From: Vladimir Popov <vladimir.popov@xored.com>
Date: Thu, 26 Nov 2020 16:03:22 +0700
Subject: [PATCH 09/11] Use VFInterfaceName only in kernel case

Signed-off-by: Vladimir Popov <vladimir.popov@xored.com>
---
 .golangci.yml                                 |   9 ++
 .../resourcepool/physical_functions.yml       |   6 -
 .../common/resourcepool/server.go             |  12 +-
 .../common/resourcepool/server_test.go        | 116 ++++++++++++------
 4 files changed, 93 insertions(+), 50 deletions(-)

diff --git a/.golangci.yml b/.golangci.yml
index 06184c9d..d872d94a 100644
--- a/.golangci.yml
+++ b/.golangci.yml
@@ -170,3 +170,12 @@ issues:
   max-issues-per-linter: 0
   max-same-issues: 0
   exclude-rules:
+    - path: ./
+      linters:
+        - gocritic
+      text: "sloppyReassign: re-assignment to `err` can be replaced with `err :="
+    - path: ./
+      linters:
+        - gocritic
+        - govet
+      text: "shadow: declaration of \"err\" shadows declaration"
diff --git a/pkg/networkservice/common/resourcepool/physical_functions.yml b/pkg/networkservice/common/resourcepool/physical_functions.yml
index 15f504a2..c73955cc 100644
--- a/pkg/networkservice/common/resourcepool/physical_functions.yml
+++ b/pkg/networkservice/common/resourcepool/physical_functions.yml
@@ -3,27 +3,21 @@
   addr: 0000:00:01.0
   ifName: pf-1
   iommuGroup: 1
-  driver: pf-1-driver
   vfs:
     - addr: 0000:00:01.1
       ifName: vf-1-1
       iommuGroup: 1
-      driver: vf-1-driver
     - addr: 0000:00:01.2
       ifName: vf-1-2
       iommuGroup: 1
-      driver: vf-1-driver
 0000:00:02.0:
   addr: 0000:00:02.0
   ifName: pf-2
   iommuGroup: 2
-  driver: pf-2-driver
   vfs:
     - addr: 0000:00:02.1
       ifName: vf-2-1
       iommuGroup: 2
-      driver: vf-2-driver
     - addr: 0000:00:02.2
       ifName: vf-2-2
       iommuGroup: 2
-      driver: vf-2-driver
diff --git a/pkg/networkservice/common/resourcepool/server.go b/pkg/networkservice/common/resourcepool/server.go
index 92d7c5b7..6b211524 100644
--- a/pkg/networkservice/common/resourcepool/server.go
+++ b/pkg/networkservice/common/resourcepool/server.go
@@ -107,7 +107,13 @@ func (s *resourcePoolServer) Request(ctx context.Context, request *networkservic
 			return err
 		}
 
-		if s.driverType == sriov.VFIOPCIDriver {
+		switch s.driverType {
+		case sriov.KernelDriver:
+			vfConfig.VFInterfaceName, err = vf.GetNetInterfaceName()
+			if err != nil {
+				return errors.Wrapf(err, "failed to get VF net interface name: %v", vf.GetPCIAddress())
+			}
+		case sriov.VFIOPCIDriver:
 			vfio.ToMechanism(request.GetConnection().GetMechanism()).SetIommuGroup(iommuGroup)
 		}
 
@@ -151,10 +157,6 @@ func (s *resourcePoolServer) selectVF(connID string, vfConfig *vfconfig.VFConfig
 			if err != nil {
 				return nil, errors.Wrapf(err, "failed to get VF: %v", vfPCIAddr)
 			}
-			vfConfig.VFInterfaceName, err = vf.GetNetInterfaceName()
-			if err != nil {
-				return nil, errors.Errorf("failed to get VF net interface name: %v", vfPCIAddr)
-			}
 
 			vfConfig.VFNum = i
 
diff --git a/pkg/networkservice/common/resourcepool/server_test.go b/pkg/networkservice/common/resourcepool/server_test.go
index 0d4476ab..7fdaad5a 100644
--- a/pkg/networkservice/common/resourcepool/server_test.go
+++ b/pkg/networkservice/common/resourcepool/server_test.go
@@ -25,6 +25,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/networkservicemesh/api/pkg/api/networkservice"
+	"github.com/networkservicemesh/api/pkg/api/networkservice/mechanisms/kernel"
 	"github.com/networkservicemesh/api/pkg/api/networkservice/mechanisms/vfio"
 	"github.com/networkservicemesh/sdk-kernel/pkg/kernel/networkservice/vfconfig"
 
@@ -40,63 +41,100 @@ const (
 	physicalFunctionsFilename = "physical_functions.yml"
 	configFileName            = "config.yml"
 	pf2PciAddr                = "0000:00:02.0"
+	vf2KernelDriver           = "vf-2-driver"
 )
 
-func Test_resourcePoolServer_Request(t *testing.T) {
-	vfConfig := &vfconfig.VFConfig{}
-	ctx := vfconfig.WithConfig(context.TODO(), vfConfig)
+type sample struct {
+	driverType sriov.DriverType
+	mechanism  string
+	test       func(t *testing.T, pfs map[string]*sriovtest.PCIPhysicalFunction, vfConfig *vfconfig.VFConfig, conn *networkservice.Connection)
+}
 
-	var pfs map[string]*sriovtest.PCIPhysicalFunction
-	_ = yamlhelper.UnmarshalFile(physicalFunctionsFilename, &pfs)
+var samples = []*sample{
+	{
+		driverType: sriov.KernelDriver,
+		mechanism:  kernel.MECHANISM,
+		test: func(t *testing.T, pfs map[string]*sriovtest.PCIPhysicalFunction, vfConfig *vfconfig.VFConfig, _ *networkservice.Connection) {
+			require.Equal(t, vf2KernelDriver, pfs[pf2PciAddr].Vfs[0].Driver)
+			require.Equal(t, vf2KernelDriver, pfs[pf2PciAddr].Vfs[1].Driver)
+
+			require.Equal(t, &vfconfig.VFConfig{
+				PFInterfaceName: pfs[pf2PciAddr].IfName,
+				VFInterfaceName: pfs[pf2PciAddr].Vfs[1].IfName,
+				VFNum:           1,
+			}, vfConfig)
+		},
+	},
+	{
+		driverType: sriov.VFIOPCIDriver,
+		mechanism:  vfio.MECHANISM,
+		test: func(t *testing.T, pfs map[string]*sriovtest.PCIPhysicalFunction, vfConfig *vfconfig.VFConfig, conn *networkservice.Connection) {
+			require.Equal(t, string(sriov.VFIOPCIDriver), pfs[pf2PciAddr].Vfs[0].Driver)
+			require.Equal(t, string(sriov.VFIOPCIDriver), pfs[pf2PciAddr].Vfs[1].Driver)
+
+			require.Equal(t, &vfconfig.VFConfig{
+				PFInterfaceName: pfs[pf2PciAddr].IfName,
+				VFNum:           1,
+			}, vfConfig)
+
+			require.Equal(t, vfio.ToMechanism(conn.Mechanism).GetIommuGroup(), pfs[pf2PciAddr].Vfs[1].IOMMUGroup)
+		},
+	},
+}
 
-	conf, err := config.ReadConfig(context.TODO(), configFileName)
-	require.NoError(t, err)
+func TestResourcePoolServer_Request(t *testing.T) {
+	for i := range samples {
+		sample := samples[i]
+		t.Run(sample.mechanism, func(t *testing.T) {
+			vfConfig := new(vfconfig.VFConfig)
+			ctx := vfconfig.WithConfig(context.TODO(), vfConfig)
 
-	pciPool, err := pci.NewTestPool(pfs, conf)
-	require.NoError(t, err)
+			var pfs map[string]*sriovtest.PCIPhysicalFunction
+			_ = yamlhelper.UnmarshalFile(physicalFunctionsFilename, &pfs)
 
-	resourcePool := &resourcePoolMock{}
+			conf, err := config.ReadConfig(context.TODO(), configFileName)
+			require.NoError(t, err)
 
-	server := resourcepool.NewServer(sriov.VFIOPCIDriver, &sync.Mutex{}, pciPool, resourcePool, conf)
+			pciPool, err := pci.NewTestPool(pfs, conf)
+			require.NoError(t, err)
 
-	// 1. Request
+			resourcePool := new(resourcePoolMock)
 
-	resourcePool.mock.On("Select", "1", sriov.VFIOPCIDriver).
-		Return(pfs[pf2PciAddr].Vfs[1].Addr, nil)
+			server := resourcepool.NewServer(sample.driverType, new(sync.Mutex), pciPool, resourcePool, conf)
 
-	conn, err := server.Request(ctx, &networkservice.NetworkServiceRequest{
-		Connection: &networkservice.Connection{
-			Id: "id",
-			Mechanism: &networkservice.Mechanism{
-				Type: vfio.MECHANISM,
-				Parameters: map[string]string{
-					resourcepool.TokenIDKey: "1",
-				},
-			},
-		},
-	})
-	require.NoError(t, err)
+			// 1. Request
 
-	resourcePool.mock.AssertNumberOfCalls(t, "Select", 1)
+			resourcePool.mock.On("Select", "1", sample.driverType).
+				Return(pfs[pf2PciAddr].Vfs[1].Addr, nil)
 
-	require.Equal(t, pfs[pf2PciAddr].Vfs[0].Driver, string(sriov.VFIOPCIDriver))
-	require.Equal(t, pfs[pf2PciAddr].Vfs[1].Driver, string(sriov.VFIOPCIDriver))
+			conn, err := server.Request(ctx, &networkservice.NetworkServiceRequest{
+				Connection: &networkservice.Connection{
+					Id: "id",
+					Mechanism: &networkservice.Mechanism{
+						Type: sample.mechanism,
+						Parameters: map[string]string{
+							resourcepool.TokenIDKey: "1",
+						},
+					},
+				},
+			})
+			require.NoError(t, err)
 
-	require.Equal(t, vfConfig.PFInterfaceName, pfs[pf2PciAddr].IfName)
-	require.Equal(t, vfConfig.VFInterfaceName, pfs[pf2PciAddr].Vfs[1].IfName)
-	require.Equal(t, vfConfig.VFNum, 1)
+			resourcePool.mock.AssertNumberOfCalls(t, "Select", 1)
 
-	require.Equal(t, vfio.ToMechanism(conn.Mechanism).GetIommuGroup(), pfs[pf2PciAddr].Vfs[1].IOMMUGroup)
+			sample.test(t, pfs, vfConfig, conn)
 
-	// 2. Close
+			// 2. Close
 
-	resourcePool.mock.On("Free", pfs[pf2PciAddr].Vfs[1].Addr).
-		Return(nil)
+			resourcePool.mock.On("Free", pfs[pf2PciAddr].Vfs[1].Addr).
+				Return(nil)
 
-	_, err = server.Close(ctx, conn)
-	require.NoError(t, err)
+			_, err = server.Close(ctx, conn)
+			require.NoError(t, err)
 
-	resourcePool.mock.AssertNumberOfCalls(t, "Free", 1)
+			resourcePool.mock.AssertNumberOfCalls(t, "Free", 1)
+		})
+	}
 }
 
 type resourcePoolMock struct {

From 9affdbcb9a7b5169ec7f3f071f8c927914389389 Mon Sep 17 00:00:00 2001
From: Vladimir Popov <vladimir.popov@xored.com>
Date: Tue, 8 Dec 2020 13:23:39 +0700
Subject: [PATCH 10/11] Use cgroupDir pattern instead of cgroupDir

Signed-off-by: Vladimir Popov <vladimir.popov@xored.com>
---
 .../common/mechanisms/vfio/client.go          |   6 +-
 .../mechanisms/vfio/client_linux_test.go      | 132 -------------
 .../{client_darwin_test.go => client_test.go} |  40 ++--
 .../common/mechanisms/vfio/client_windows.go  |  43 ----
 .../common/mechanisms/vfio/const_test.go      |   2 -
 .../vfio/{server_linux.go => server.go}       |  78 +++++---
 .../common/mechanisms/vfio/server_darwin.go   | 185 ------------------
 .../common/mechanisms/vfio/server_test.go     |  55 +++---
 .../common/mechanisms/vfio/tools_darwin.go    |  29 +++
 .../common/mechanisms/vfio/tools_linux.go     |  29 +++
 10 files changed, 159 insertions(+), 440 deletions(-)
 delete mode 100644 pkg/networkservice/common/mechanisms/vfio/client_linux_test.go
 rename pkg/networkservice/common/mechanisms/vfio/{client_darwin_test.go => client_test.go} (80%)
 delete mode 100644 pkg/networkservice/common/mechanisms/vfio/client_windows.go
 rename pkg/networkservice/common/mechanisms/vfio/{server_linux.go => server.go} (65%)
 delete mode 100644 pkg/networkservice/common/mechanisms/vfio/server_darwin.go
 create mode 100644 pkg/networkservice/common/mechanisms/vfio/tools_darwin.go
 create mode 100644 pkg/networkservice/common/mechanisms/vfio/tools_linux.go

diff --git a/pkg/networkservice/common/mechanisms/vfio/client.go b/pkg/networkservice/common/mechanisms/vfio/client.go
index 16f36fcb..5756fddf 100644
--- a/pkg/networkservice/common/mechanisms/vfio/client.go
+++ b/pkg/networkservice/common/mechanisms/vfio/client.go
@@ -21,7 +21,7 @@ package vfio
 import (
 	"context"
 	"os"
-	"path"
+	"path/filepath"
 
 	"github.com/golang/protobuf/ptypes/empty"
 	"golang.org/x/sys/unix"
@@ -68,7 +68,7 @@ func (c *vfioClient) Request(ctx context.Context, request *networkservice.Networ
 		}
 
 		if err := unix.Mknod(
-			path.Join(c.vfioDir, vfioDevice),
+			filepath.Join(c.vfioDir, vfioDevice),
 			unix.S_IFCHR|mknodPerm,
 			int(unix.Mkdev(mech.GetVfioMajor(), mech.GetVfioMinor())),
 		); err != nil && !os.IsExist(err) {
@@ -78,7 +78,7 @@ func (c *vfioClient) Request(ctx context.Context, request *networkservice.Networ
 
 		igid := mech.GetParameters()[vfio.IommuGroupKey]
 		if err := unix.Mknod(
-			path.Join(c.vfioDir, igid),
+			filepath.Join(c.vfioDir, igid),
 			unix.S_IFCHR|mknodPerm,
 			int(unix.Mkdev(mech.GetDeviceMajor(), mech.GetDeviceMinor())),
 		); err != nil && !os.IsExist(err) {
diff --git a/pkg/networkservice/common/mechanisms/vfio/client_linux_test.go b/pkg/networkservice/common/mechanisms/vfio/client_linux_test.go
deleted file mode 100644
index 6a5914f8..00000000
--- a/pkg/networkservice/common/mechanisms/vfio/client_linux_test.go
+++ /dev/null
@@ -1,132 +0,0 @@
-// Copyright (c) 2020 Doc.ai and/or its affiliates.
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at:
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package vfio_test
-
-import (
-	"context"
-	"net/url"
-	"os"
-	"path"
-	"testing"
-	"time"
-
-	"golang.org/x/sys/unix"
-	"google.golang.org/grpc"
-
-	"github.com/golang/protobuf/ptypes/empty"
-	"github.com/networkservicemesh/api/pkg/api/networkservice"
-	vfiomech "github.com/networkservicemesh/api/pkg/api/networkservice/mechanisms/vfio"
-	"github.com/networkservicemesh/sdk/pkg/networkservice/common/mechanisms"
-	"github.com/networkservicemesh/sdk/pkg/networkservice/core/chain"
-	"github.com/networkservicemesh/sdk/pkg/networkservice/core/next"
-	"github.com/networkservicemesh/sdk/pkg/tools/grpcutils"
-	"github.com/stretchr/testify/assert"
-
-	"github.com/networkservicemesh/sdk-sriov/pkg/networkservice/common/mechanisms/vfio"
-)
-
-const (
-	serverSocket = "server.socket"
-)
-
-func testServer(ctx context.Context, tmpDir string) (grpc.ClientConnInterface, error) {
-	socketURL := &url.URL{
-		Scheme: "unix",
-		Path:   path.Join(tmpDir, serverSocket),
-	}
-
-	server := grpc.NewServer()
-	networkservice.RegisterNetworkServiceServer(server, mechanisms.NewServer(map[string]networkservice.NetworkServiceServer{
-		vfiomech.MECHANISM: &vfioForwarderStub{
-			iommuGroup:  iommuGroup,
-			vfioMajor:   1,
-			vfioMinor:   2,
-			deviceMajor: 3,
-			deviceMinor: 4,
-		},
-	}))
-	_ = grpcutils.ListenAndServe(ctx, socketURL, server)
-
-	<-time.After(1 * time.Millisecond) // wait for the server to start
-
-	return grpc.DialContext(ctx, socketURL.String(), grpc.WithInsecure())
-}
-
-func TestVfioClient_Request(t *testing.T) {
-	ctx, cancel := context.WithTimeout(context.TODO(), 10*time.Second)
-	defer cancel()
-
-	tmpDir := path.Join(os.TempDir(), t.Name())
-	err := os.MkdirAll(tmpDir, 0750)
-	assert.Nil(t, err)
-	defer func() { _ = os.RemoveAll(tmpDir) }()
-
-	cc, err := testServer(ctx, tmpDir)
-	assert.Nil(t, err)
-
-	client := chain.NewNetworkServiceClient(
-		vfio.NewClient(tmpDir, cgroupDir),
-		networkservice.NewNetworkServiceClient(cc),
-	)
-
-	conn, err := client.Request(ctx, &networkservice.NetworkServiceRequest{
-		Connection: &networkservice.Connection{},
-	})
-	assert.Nil(t, err)
-
-	mech := vfiomech.ToMechanism(conn.GetMechanism())
-	assert.NotNil(t, mech)
-	assert.Equal(t, cgroupDir, mech.GetCgroupDir())
-
-	info := new(unix.Stat_t)
-
-	err = unix.Stat(path.Join(tmpDir, vfioDevice), info)
-	assert.Nil(t, err)
-	assert.Equal(t, uint32(1), unix.Major(info.Rdev))
-	assert.Equal(t, uint32(2), unix.Minor(info.Rdev))
-
-	err = unix.Stat(path.Join(tmpDir, iommuGroupString), info)
-	assert.Nil(t, err)
-	assert.Equal(t, uint32(3), unix.Major(info.Rdev))
-	assert.Equal(t, uint32(4), unix.Minor(info.Rdev))
-
-	assert.Nil(t, ctx.Err())
-}
-
-type vfioForwarderStub struct {
-	iommuGroup  uint
-	vfioMajor   uint32
-	vfioMinor   uint32
-	deviceMajor uint32
-	deviceMinor uint32
-}
-
-func (vf *vfioForwarderStub) Request(ctx context.Context, request *networkservice.NetworkServiceRequest) (*networkservice.Connection, error) {
-	if mech := vfiomech.ToMechanism(request.GetConnection().GetMechanism()); mech != nil {
-		mech.SetIommuGroup(vf.iommuGroup)
-		mech.SetVfioMajor(vf.vfioMajor)
-		mech.SetVfioMinor(vf.vfioMinor)
-		mech.SetDeviceMajor(vf.deviceMajor)
-		mech.SetDeviceMinor(vf.deviceMinor)
-	}
-
-	return next.Server(ctx).Request(ctx, request)
-}
-
-func (vf *vfioForwarderStub) Close(ctx context.Context, conn *networkservice.Connection) (*empty.Empty, error) {
-	return next.Server(ctx).Close(ctx, conn)
-}
diff --git a/pkg/networkservice/common/mechanisms/vfio/client_darwin_test.go b/pkg/networkservice/common/mechanisms/vfio/client_test.go
similarity index 80%
rename from pkg/networkservice/common/mechanisms/vfio/client_darwin_test.go
rename to pkg/networkservice/common/mechanisms/vfio/client_test.go
index 3562e05a..f1e897be 100644
--- a/pkg/networkservice/common/mechanisms/vfio/client_darwin_test.go
+++ b/pkg/networkservice/common/mechanisms/vfio/client_test.go
@@ -14,16 +14,19 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//+build !windows
+
 package vfio_test
 
 import (
 	"context"
 	"net/url"
 	"os"
-	"path"
+	"path/filepath"
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/require"
 	"golang.org/x/sys/unix"
 	"google.golang.org/grpc"
 
@@ -34,7 +37,6 @@ import (
 	"github.com/networkservicemesh/sdk/pkg/networkservice/core/chain"
 	"github.com/networkservicemesh/sdk/pkg/networkservice/core/next"
 	"github.com/networkservicemesh/sdk/pkg/tools/grpcutils"
-	"github.com/stretchr/testify/assert"
 
 	"github.com/networkservicemesh/sdk-sriov/pkg/networkservice/common/mechanisms/vfio"
 )
@@ -46,7 +48,7 @@ const (
 func testServer(ctx context.Context, tmpDir string) (grpc.ClientConnInterface, error) {
 	socketURL := &url.URL{
 		Scheme: "unix",
-		Path:   path.Join(tmpDir, serverSocket),
+		Path:   filepath.Join(tmpDir, serverSocket),
 	}
 
 	server := grpc.NewServer()
@@ -66,17 +68,17 @@ func testServer(ctx context.Context, tmpDir string) (grpc.ClientConnInterface, e
 	return grpc.DialContext(ctx, socketURL.String(), grpc.WithInsecure())
 }
 
-func TestVfioClient_Request(t *testing.T) {
+func TestVFIOClient_Request(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.TODO(), 10*time.Second)
 	defer cancel()
 
-	tmpDir := path.Join(os.TempDir(), t.Name())
+	tmpDir := filepath.Join(os.TempDir(), t.Name())
 	err := os.MkdirAll(tmpDir, 0750)
-	assert.Nil(t, err)
+	require.NoError(t, err)
 	defer func() { _ = os.RemoveAll(tmpDir) }()
 
 	cc, err := testServer(ctx, tmpDir)
-	assert.Nil(t, err)
+	require.NoError(t, err)
 
 	client := chain.NewNetworkServiceClient(
 		vfio.NewClient(tmpDir, cgroupDir),
@@ -86,25 +88,25 @@ func TestVfioClient_Request(t *testing.T) {
 	conn, err := client.Request(ctx, &networkservice.NetworkServiceRequest{
 		Connection: &networkservice.Connection{},
 	})
-	assert.Nil(t, err)
+	require.NoError(t, err)
 
 	mech := vfiomech.ToMechanism(conn.GetMechanism())
-	assert.NotNil(t, mech)
-	assert.Equal(t, cgroupDir, mech.GetCgroupDir())
+	require.NotNil(t, mech)
+	require.Equal(t, cgroupDir, mech.GetCgroupDir())
 
 	info := new(unix.Stat_t)
 
-	err = unix.Stat(path.Join(tmpDir, vfioDevice), info)
-	assert.Nil(t, err)
-	assert.Equal(t, uint32(1), unix.Major(uint64(info.Rdev)))
-	assert.Equal(t, uint32(2), unix.Minor(uint64(info.Rdev)))
+	err = unix.Stat(filepath.Join(tmpDir, vfioDevice), info)
+	require.NoError(t, err)
+	require.Equal(t, uint32(1), vfio.Major(info.Rdev))
+	require.Equal(t, uint32(2), vfio.Minor(info.Rdev))
 
-	err = unix.Stat(path.Join(tmpDir, iommuGroupString), info)
-	assert.Nil(t, err)
-	assert.Equal(t, uint32(3), unix.Major(uint64(info.Rdev)))
-	assert.Equal(t, uint32(4), unix.Minor(uint64(info.Rdev)))
+	err = unix.Stat(filepath.Join(tmpDir, iommuGroupString), info)
+	require.NoError(t, err)
+	require.Equal(t, uint32(3), vfio.Major(info.Rdev))
+	require.Equal(t, uint32(4), vfio.Minor(info.Rdev))
 
-	assert.Nil(t, ctx.Err())
+	require.NoError(t, ctx.Err())
 }
 
 type vfioForwarderStub struct {
diff --git a/pkg/networkservice/common/mechanisms/vfio/client_windows.go b/pkg/networkservice/common/mechanisms/vfio/client_windows.go
deleted file mode 100644
index 2266fb50..00000000
--- a/pkg/networkservice/common/mechanisms/vfio/client_windows.go
+++ /dev/null
@@ -1,43 +0,0 @@
-// Copyright (c) 2020 Doc.ai and/or its affiliates.
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at:
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package vfio
-
-import (
-	"context"
-
-	"github.com/golang/protobuf/ptypes/empty"
-	"github.com/pkg/errors"
-	"google.golang.org/grpc"
-
-	"github.com/networkservicemesh/api/pkg/api/networkservice"
-	"github.com/networkservicemesh/sdk/pkg/networkservice/core/next"
-)
-
-type vfioClient struct{}
-
-// NewClient returns a new VFIO client chain element
-func NewClient(_, _ string) networkservice.NetworkServiceClient {
-	return &vfioClient{}
-}
-
-func (c *vfioClient) Request(_ context.Context, _ *networkservice.NetworkServiceRequest, _ ...grpc.CallOption) (*networkservice.Connection, error) {
-	return nil, errors.New("VFIO is not supported for Windows")
-}
-
-func (c *vfioClient) Close(ctx context.Context, conn *networkservice.Connection, opts ...grpc.CallOption) (*empty.Empty, error) {
-	return next.Client(ctx).Close(ctx, conn, opts...)
-}
diff --git a/pkg/networkservice/common/mechanisms/vfio/const_test.go b/pkg/networkservice/common/mechanisms/vfio/const_test.go
index 5140859b..faa6e5c4 100644
--- a/pkg/networkservice/common/mechanisms/vfio/const_test.go
+++ b/pkg/networkservice/common/mechanisms/vfio/const_test.go
@@ -14,8 +14,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-//+build !windows
-
 package vfio_test
 
 const (
diff --git a/pkg/networkservice/common/mechanisms/vfio/server_linux.go b/pkg/networkservice/common/mechanisms/vfio/server.go
similarity index 65%
rename from pkg/networkservice/common/mechanisms/vfio/server_linux.go
rename to pkg/networkservice/common/mechanisms/vfio/server.go
index a3673cc8..57a6711b 100644
--- a/pkg/networkservice/common/mechanisms/vfio/server_linux.go
+++ b/pkg/networkservice/common/mechanisms/vfio/server.go
@@ -14,6 +14,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//+build !windows
+
 // Package vfio provides server, vfioClient chain elements for the VFIO mechanism connection
 package vfio
 
@@ -21,10 +23,11 @@ import (
 	"context"
 	"fmt"
 	"io/ioutil"
-	"path"
+	"path/filepath"
 	"sync"
 
 	"github.com/golang/protobuf/ptypes/empty"
+	"github.com/pkg/errors"
 	"golang.org/x/sys/unix"
 
 	"github.com/networkservicemesh/api/pkg/api/networkservice"
@@ -59,35 +62,35 @@ func (s *vfioServer) Request(ctx context.Context, request *networkservice.Networ
 	logEntry := log.Entry(ctx).WithField("vfioServer", "Request")
 
 	if mech := vfio.ToMechanism(request.GetConnection().GetMechanism()); mech != nil {
-		vfioMajor, vfioMinor, err := s.getDeviceNumbers(path.Join(s.vfioDir, vfioDevice))
+		vfioMajor, vfioMinor, err := s.getDeviceNumbers(filepath.Join(s.vfioDir, vfioDevice))
 		if err != nil {
 			logEntry.Errorf("failed to get device numbers for the device: %v", vfioDevice)
 			return nil, err
 		}
 
 		igid := mech.GetParameters()[vfio.IommuGroupKey]
-		deviceMajor, deviceMinor, err := s.getDeviceNumbers(path.Join(s.vfioDir, igid))
+		deviceMajor, deviceMinor, err := s.getDeviceNumbers(filepath.Join(s.vfioDir, igid))
 		if err != nil {
 			logEntry.Errorf("failed to get device numbers for the device: %v", igid)
 			return nil, err
 		}
 
-		cgroupDir := path.Join(s.cgroupBaseDir, mech.GetCgroupDir())
+		cgroupDirPattern := filepath.Join(s.cgroupBaseDir, mech.GetCgroupDir())
 
 		if err := func() error {
 			s.lock.Lock()
 			defer s.lock.Unlock()
 
-			if err := s.deviceAllow(cgroupDir, vfioMajor, vfioMinor); err != nil {
+			if err := s.deviceAllow(cgroupDirPattern, vfioMajor, vfioMinor); err != nil {
 				logEntry.Errorf("failed to allow device for the client: %v", vfioDevice)
 				return err
 			}
 			mech.SetVfioMajor(vfioMajor)
 			mech.SetVfioMinor(vfioMinor)
 
-			if err := s.deviceAllow(cgroupDir, deviceMajor, deviceMinor); err != nil {
+			if err := s.deviceAllow(cgroupDirPattern, deviceMajor, deviceMinor); err != nil {
 				logEntry.Errorf("failed to allow device for the client: %v", igid)
-				_ = s.deviceDeny(cgroupDir, vfioMajor, vfioMinor)
+				_ = s.deviceDeny(cgroupDirPattern, vfioMajor, vfioMinor)
 				return err
 			}
 			mech.SetDeviceMajor(deviceMajor)
@@ -113,22 +116,30 @@ func (s *vfioServer) getDeviceNumbers(deviceFile string) (major, minor uint32, e
 	if err := unix.Stat(deviceFile, info); err != nil {
 		return 0, 0, err
 	}
-	return unix.Major(info.Rdev), unix.Minor(info.Rdev), nil
+	return Major(info.Rdev), Minor(info.Rdev), nil
 }
 
-func (s *vfioServer) deviceAllow(cgroupDir string, major, minor uint32) error {
-	key := deviceKey(cgroupDir, major, minor)
-	if counter, ok := s.deviceCounters[key]; ok && counter > 0 {
-		s.deviceCounters[key] = counter + 1
-		return nil
+func (s *vfioServer) deviceAllow(cgroupDirPattern string, major, minor uint32) error {
+	deviceFiles, err := filepath.Glob(filepath.Join(cgroupDirPattern, deviceAllowFile))
+	if err != nil || len(deviceFiles) == 0 {
+		return errors.Wrapf(err, "no cgroupDir found: %s", cgroupDirPattern)
 	}
 
-	deviceString := fmt.Sprintf(deviceStringFormat, major, minor)
-	if err := ioutil.WriteFile(path.Join(cgroupDir, deviceAllowFile), []byte(deviceString), 0); err != nil {
-		return err
-	}
+	for _, deviceFile := range deviceFiles {
+		fmt.Printf("file: %s\n", deviceFile)
+		key := deviceKey(filepath.Dir(deviceFile), major, minor)
+		if counter, ok := s.deviceCounters[key]; ok && counter > 0 {
+			s.deviceCounters[key] = counter + 1
+			return nil
+		}
 
-	s.deviceCounters[key] = 1
+		deviceString := fmt.Sprintf(deviceStringFormat, major, minor)
+		if err := ioutil.WriteFile(deviceFile, []byte(deviceString), 0); err != nil {
+			return err
+		}
+
+		s.deviceCounters[key] = 1
+	}
 
 	return nil
 }
@@ -146,7 +157,7 @@ func (s *vfioServer) close(ctx context.Context, conn *networkservice.Connection)
 	logEntry := log.Entry(ctx).WithField("vfioServer", "close")
 
 	if mech := vfio.ToMechanism(conn.GetMechanism()); mech != nil {
-		cgroupDir := path.Join(s.cgroupBaseDir, mech.GetCgroupDir())
+		cgroupDirPattern := filepath.Join(s.cgroupBaseDir, mech.GetCgroupDir())
 
 		s.lock.Lock()
 		defer s.lock.Unlock()
@@ -154,7 +165,7 @@ func (s *vfioServer) close(ctx context.Context, conn *networkservice.Connection)
 		vfioMajor := mech.GetVfioMajor()
 		vfioMinor := mech.GetVfioMinor()
 		if !(vfioMajor == 0 && vfioMinor == 0) {
-			if err := s.deviceDeny(cgroupDir, vfioMajor, vfioMinor); err != nil {
+			if err := s.deviceDeny(cgroupDirPattern, vfioMajor, vfioMinor); err != nil {
 				logEntry.Warnf("failed to deny device for the client: %v", vfioDevice)
 			}
 		}
@@ -162,22 +173,33 @@ func (s *vfioServer) close(ctx context.Context, conn *networkservice.Connection)
 		deviceMajor := mech.GetDeviceMajor()
 		deviceMinor := mech.GetDeviceMinor()
 		if !(deviceMajor == 0 && deviceMinor == 0) {
-			if err := s.deviceDeny(cgroupDir, deviceMajor, deviceMinor); err != nil {
+			if err := s.deviceDeny(cgroupDirPattern, deviceMajor, deviceMinor); err != nil {
 				logEntry.Warnf("failed to deny device for the client: %v", mech.GetIommuGroup())
 			}
 		}
 	}
 }
 
-func (s *vfioServer) deviceDeny(cgroupDir string, major, minor uint32) error {
-	key := deviceKey(cgroupDir, major, minor)
-	s.deviceCounters[key]--
-	if s.deviceCounters[key] > 0 {
-		return nil
+func (s *vfioServer) deviceDeny(cgroupDirPattern string, major, minor uint32) error {
+	deviceFiles, err := filepath.Glob(filepath.Join(cgroupDirPattern, deviceDenyFile))
+	if err != nil || len(deviceFiles) == 0 {
+		return errors.Wrapf(err, "no cgroupDir found: %s", cgroupDirPattern)
 	}
 
-	deviceString := fmt.Sprintf(deviceStringFormat, major, minor)
-	return ioutil.WriteFile(path.Join(cgroupDir, deviceDenyFile), []byte(deviceString), 0)
+	for _, deviceFile := range deviceFiles {
+		key := deviceKey(filepath.Dir(deviceFile), major, minor)
+		s.deviceCounters[key]--
+		if s.deviceCounters[key] > 0 {
+			return nil
+		}
+
+		deviceString := fmt.Sprintf(deviceStringFormat, major, minor)
+		if err := ioutil.WriteFile(deviceFile, []byte(deviceString), 0); err != nil {
+			return err
+		}
+	}
+
+	return nil
 }
 
 func deviceKey(cgroupDir string, major, minor uint32) string {
diff --git a/pkg/networkservice/common/mechanisms/vfio/server_darwin.go b/pkg/networkservice/common/mechanisms/vfio/server_darwin.go
deleted file mode 100644
index 273213b8..00000000
--- a/pkg/networkservice/common/mechanisms/vfio/server_darwin.go
+++ /dev/null
@@ -1,185 +0,0 @@
-// Copyright (c) 2020 Doc.ai and/or its affiliates.
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at:
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// Package vfio provides server, vfioClient chain elements for the VFIO mechanism connection
-package vfio
-
-import (
-	"context"
-	"fmt"
-	"io/ioutil"
-	"path"
-	"sync"
-
-	"github.com/golang/protobuf/ptypes/empty"
-	"golang.org/x/sys/unix"
-
-	"github.com/networkservicemesh/api/pkg/api/networkservice"
-	"github.com/networkservicemesh/api/pkg/api/networkservice/mechanisms/vfio"
-	"github.com/networkservicemesh/sdk/pkg/networkservice/core/next"
-	"github.com/networkservicemesh/sdk/pkg/tools/log"
-)
-
-const (
-	deviceAllowFile    = "devices.allow"
-	deviceDenyFile     = "devices.deny"
-	deviceStringFormat = "c %v:%v rwm\n"
-)
-
-type vfioServer struct {
-	vfioDir        string
-	cgroupBaseDir  string
-	deviceCounters map[string]int
-	lock           sync.Mutex
-}
-
-// NewServer returns a new VFIO server chain element
-func NewServer(vfioDir, cgroupBaseDir string) networkservice.NetworkServiceServer {
-	return &vfioServer{
-		vfioDir:        vfioDir,
-		cgroupBaseDir:  cgroupBaseDir,
-		deviceCounters: map[string]int{},
-	}
-}
-
-func (s *vfioServer) Request(ctx context.Context, request *networkservice.NetworkServiceRequest) (*networkservice.Connection, error) {
-	logEntry := log.Entry(ctx).WithField("vfioServer", "Request")
-
-	if mech := vfio.ToMechanism(request.GetConnection().GetMechanism()); mech != nil {
-		vfioMajor, vfioMinor, err := s.getDeviceNumbers(path.Join(s.vfioDir, vfioDevice))
-		if err != nil {
-			logEntry.Errorf("failed to get device numbers for the device: %v", vfioDevice)
-			return nil, err
-		}
-
-		igid := mech.GetParameters()[vfio.IommuGroupKey]
-		deviceMajor, deviceMinor, err := s.getDeviceNumbers(path.Join(s.vfioDir, igid))
-		if err != nil {
-			logEntry.Errorf("failed to get device numbers for the device: %v", igid)
-			return nil, err
-		}
-
-		cgroupDir := path.Join(s.cgroupBaseDir, mech.GetCgroupDir())
-
-		if err := func() error {
-			s.lock.Lock()
-			defer s.lock.Unlock()
-
-			if err := s.deviceAllow(cgroupDir, vfioMajor, vfioMinor); err != nil {
-				logEntry.Errorf("failed to allow device for the client: %v", vfioDevice)
-				return err
-			}
-			mech.SetVfioMajor(vfioMajor)
-			mech.SetVfioMinor(vfioMinor)
-
-			if err := s.deviceAllow(cgroupDir, deviceMajor, deviceMinor); err != nil {
-				logEntry.Errorf("failed to allow device for the client: %v", igid)
-				_ = s.deviceDeny(cgroupDir, vfioMajor, vfioMinor)
-				return err
-			}
-			mech.SetDeviceMajor(deviceMajor)
-			mech.SetDeviceMinor(deviceMinor)
-
-			return nil
-		}(); err != nil {
-			return nil, err
-		}
-	}
-
-	conn, err := next.Server(ctx).Request(ctx, request)
-	if err != nil {
-		s.close(ctx, request.GetConnection())
-		return nil, err
-	}
-
-	return conn, nil
-}
-
-func (s *vfioServer) getDeviceNumbers(deviceFile string) (major, minor uint32, err error) {
-	info := new(unix.Stat_t)
-	if err := unix.Stat(deviceFile, info); err != nil {
-		return 0, 0, err
-	}
-	return unix.Major(uint64(info.Rdev)), unix.Minor(uint64(info.Rdev)), nil
-}
-
-func (s *vfioServer) deviceAllow(cgroupDir string, major, minor uint32) error {
-	key := deviceKey(cgroupDir, major, minor)
-	if counter, ok := s.deviceCounters[key]; ok && counter > 0 {
-		s.deviceCounters[key] = counter + 1
-		return nil
-	}
-
-	deviceString := fmt.Sprintf(deviceStringFormat, major, minor)
-	if err := ioutil.WriteFile(path.Join(cgroupDir, deviceAllowFile), []byte(deviceString), 0); err != nil {
-		return err
-	}
-
-	s.deviceCounters[key] = 1
-
-	return nil
-}
-
-func (s *vfioServer) Close(ctx context.Context, conn *networkservice.Connection) (*empty.Empty, error) {
-	s.close(ctx, conn)
-
-	if _, err := next.Server(ctx).Close(ctx, conn); err != nil {
-		return nil, err
-	}
-	return &empty.Empty{}, nil
-}
-
-func (s *vfioServer) close(ctx context.Context, conn *networkservice.Connection) {
-	logEntry := log.Entry(ctx).WithField("vfioServer", "close")
-
-	if mech := vfio.ToMechanism(conn.GetMechanism()); mech != nil {
-		cgroupDir := path.Join(s.cgroupBaseDir, mech.GetCgroupDir())
-
-		s.lock.Lock()
-		defer s.lock.Unlock()
-
-		vfioMajor := mech.GetVfioMajor()
-		vfioMinor := mech.GetVfioMinor()
-		if !(vfioMajor == 0 && vfioMinor == 0) {
-			if err := s.deviceDeny(cgroupDir, vfioMajor, vfioMinor); err != nil {
-				logEntry.Warnf("failed to deny device for the client: %v", vfioDevice)
-			}
-		}
-
-		deviceMajor := mech.GetDeviceMajor()
-		deviceMinor := mech.GetDeviceMinor()
-		if !(deviceMajor == 0 && deviceMinor == 0) {
-			if err := s.deviceDeny(cgroupDir, deviceMajor, deviceMinor); err != nil {
-				logEntry.Warnf("failed to deny device for the client: %v", mech.GetIommuGroup())
-			}
-		}
-	}
-}
-
-func (s *vfioServer) deviceDeny(cgroupDir string, major, minor uint32) error {
-	key := deviceKey(cgroupDir, major, minor)
-	s.deviceCounters[key]--
-	if s.deviceCounters[key] > 0 {
-		return nil
-	}
-
-	deviceString := fmt.Sprintf(deviceStringFormat, major, minor)
-	return ioutil.WriteFile(path.Join(cgroupDir, deviceDenyFile), []byte(deviceString), 0)
-}
-
-func deviceKey(cgroupDir string, major, minor uint32) string {
-	return fmt.Sprintf("%s:%d:%d", cgroupDir, major, minor)
-}
diff --git a/pkg/networkservice/common/mechanisms/vfio/server_test.go b/pkg/networkservice/common/mechanisms/vfio/server_test.go
index 6cb418ce..87674814 100644
--- a/pkg/networkservice/common/mechanisms/vfio/server_test.go
+++ b/pkg/networkservice/common/mechanisms/vfio/server_test.go
@@ -22,20 +22,19 @@ import (
 	"context"
 	"fmt"
 	"os"
-	"path"
+	"path/filepath"
 	"reflect"
 	"sync"
 	"testing"
 	"time"
 
-	"golang.org/x/sys/unix"
-
 	"github.com/networkservicemesh/api/pkg/api/networkservice"
 	"github.com/networkservicemesh/api/pkg/api/networkservice/mechanisms/cls"
 	vfiomech "github.com/networkservicemesh/api/pkg/api/networkservice/mechanisms/vfio"
 	"github.com/networkservicemesh/sdk/pkg/networkservice/common/mechanisms"
 	"github.com/networkservicemesh/sdk/pkg/networkservice/core/chain"
-	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/sys/unix"
 
 	"github.com/networkservicemesh/sdk-sriov/pkg/networkservice/common/mechanisms/vfio"
 	"github.com/networkservicemesh/sdk-sriov/pkg/sriov/sriovtest"
@@ -48,9 +47,9 @@ const (
 )
 
 func testVFIOServer(ctx context.Context, t *testing.T, allowedDevices *allowedDevices) (server networkservice.NetworkServiceServer, tmpDir string) {
-	tmpDir = path.Join(os.TempDir(), t.Name())
-	err := os.MkdirAll(path.Join(tmpDir, cgroupDir), 0750)
-	assert.Nil(t, err)
+	tmpDir = filepath.Join(os.TempDir(), t.Name())
+	err := os.MkdirAll(filepath.Join(tmpDir, cgroupDir), 0750)
+	require.NoError(t, err)
 
 	server = chain.NewNetworkServiceServer(
 		mechanisms.NewServer(map[string]networkservice.NetworkServiceServer{
@@ -58,22 +57,22 @@ func testVFIOServer(ctx context.Context, t *testing.T, allowedDevices *allowedDe
 		}),
 	)
 
-	err = sriovtest.InputFileAPI(ctx, path.Join(tmpDir, cgroupDir, deviceAllowFile), func(s string) {
+	err = sriovtest.InputFileAPI(ctx, filepath.Join(tmpDir, cgroupDir, deviceAllowFile), func(s string) {
 		var major, minor int
 		_, _ = fmt.Sscanf(s, deviceStringFormat, &major, &minor)
 		allowedDevices.Lock()
 		allowedDevices.devices[fmt.Sprintf("%d:%d", major, minor)] = true
 		allowedDevices.Unlock()
 	})
-	assert.Nil(t, err)
-	err = sriovtest.InputFileAPI(ctx, path.Join(tmpDir, cgroupDir, deviceDenyFile), func(s string) {
+	require.NoError(t, err)
+	err = sriovtest.InputFileAPI(ctx, filepath.Join(tmpDir, cgroupDir, deviceDenyFile), func(s string) {
 		var major, minor int
 		_, _ = fmt.Sscanf(s, deviceStringFormat, &major, &minor)
 		allowedDevices.Lock()
 		delete(allowedDevices.devices, fmt.Sprintf("%d:%d", major, minor))
 		allowedDevices.Unlock()
 	})
-	assert.Nil(t, err)
+	require.NoError(t, err)
 
 	return server, tmpDir
 }
@@ -88,10 +87,10 @@ func TestVFIOServer_Request(t *testing.T) {
 	server, tmpDir := testVFIOServer(ctx, t, allowedDevices)
 	defer func() { _ = os.RemoveAll(tmpDir) }()
 
-	err := unix.Mknod(path.Join(tmpDir, vfioDevice), unix.S_IFCHR|0666, int(unix.Mkdev(1, 2)))
-	assert.Nil(t, err)
-	err = unix.Mknod(path.Join(tmpDir, iommuGroupString), unix.S_IFCHR|0666, int(unix.Mkdev(3, 4)))
-	assert.Nil(t, err)
+	err := unix.Mknod(filepath.Join(tmpDir, vfioDevice), unix.S_IFCHR|0666, int(unix.Mkdev(1, 2)))
+	require.NoError(t, err)
+	err = unix.Mknod(filepath.Join(tmpDir, iommuGroupString), unix.S_IFCHR|0666, int(unix.Mkdev(3, 4)))
+	require.NoError(t, err)
 
 	conn, err := server.Request(ctx, &networkservice.NetworkServiceRequest{
 		Connection: &networkservice.Connection{},
@@ -100,22 +99,22 @@ func TestVFIOServer_Request(t *testing.T) {
 				Cls:  cls.LOCAL,
 				Type: vfiomech.MECHANISM,
 				Parameters: map[string]string{
-					vfiomech.CgroupDirKey:  cgroupDir,
+					vfiomech.CgroupDirKey:  "*",
 					vfiomech.IommuGroupKey: iommuGroupString,
 				},
 			},
 		},
 	})
-	assert.Nil(t, err)
+	require.NoError(t, err)
 
 	mech := vfiomech.ToMechanism(conn.GetMechanism())
-	assert.NotNil(t, mech)
-	assert.Equal(t, uint32(1), mech.GetVfioMajor())
-	assert.Equal(t, uint32(2), mech.GetVfioMinor())
-	assert.Equal(t, uint32(3), mech.GetDeviceMajor())
-	assert.Equal(t, uint32(4), mech.GetDeviceMinor())
+	require.NotNil(t, mech)
+	require.Equal(t, uint32(1), mech.GetVfioMajor())
+	require.Equal(t, uint32(2), mech.GetVfioMinor())
+	require.Equal(t, uint32(3), mech.GetDeviceMajor())
+	require.Equal(t, uint32(4), mech.GetDeviceMinor())
 
-	assert.Eventually(t, func() bool {
+	require.Eventually(t, func() bool {
 		allowedDevices.Lock()
 		defer allowedDevices.Unlock()
 		return reflect.DeepEqual(map[string]bool{
@@ -124,7 +123,7 @@ func TestVFIOServer_Request(t *testing.T) {
 		}, allowedDevices.devices)
 	}, time.Second, 10*time.Millisecond)
 
-	assert.Nil(t, ctx.Err())
+	require.NoError(t, ctx.Err())
 }
 
 func TestVFIOServer_Close(t *testing.T) {
@@ -145,7 +144,7 @@ func TestVFIOServer_Close(t *testing.T) {
 			Cls:  cls.LOCAL,
 			Type: vfiomech.MECHANISM,
 			Parameters: map[string]string{
-				vfiomech.CgroupDirKey:   cgroupDir,
+				vfiomech.CgroupDirKey:   "*",
 				vfiomech.IommuGroupKey:  iommuGroupString,
 				vfiomech.VfioMajorKey:   "1",
 				vfiomech.VfioMinorKey:   "2",
@@ -156,15 +155,15 @@ func TestVFIOServer_Close(t *testing.T) {
 	}
 
 	_, err := server.Close(ctx, conn)
-	assert.Nil(t, err)
+	require.NoError(t, err)
 
-	assert.Eventually(t, func() bool {
+	require.Eventually(t, func() bool {
 		allowedDevices.Lock()
 		defer allowedDevices.Unlock()
 		return reflect.DeepEqual(map[string]bool{}, allowedDevices.devices)
 	}, time.Second, 10*time.Millisecond)
 
-	assert.Nil(t, ctx.Err())
+	require.NoError(t, ctx.Err())
 }
 
 type allowedDevices struct {
diff --git a/pkg/networkservice/common/mechanisms/vfio/tools_darwin.go b/pkg/networkservice/common/mechanisms/vfio/tools_darwin.go
new file mode 100644
index 00000000..35bccd33
--- /dev/null
+++ b/pkg/networkservice/common/mechanisms/vfio/tools_darwin.go
@@ -0,0 +1,29 @@
+// Copyright (c) 2020 Doc.ai and/or its affiliates.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package vfio
+
+import "golang.org/x/sys/unix"
+
+// Major is unix.Major typed to unix.Stat_t .Dev
+func Major(dev int32) uint32 {
+	return unix.Major(uint64(dev))
+}
+
+// Minor is unix.Minor typed to unix.Stat_t .Dev
+func Minor(dev int32) uint32 {
+	return unix.Minor(uint64(dev))
+}
diff --git a/pkg/networkservice/common/mechanisms/vfio/tools_linux.go b/pkg/networkservice/common/mechanisms/vfio/tools_linux.go
new file mode 100644
index 00000000..2b69e7a2
--- /dev/null
+++ b/pkg/networkservice/common/mechanisms/vfio/tools_linux.go
@@ -0,0 +1,29 @@
+// Copyright (c) 2020 Doc.ai and/or its affiliates.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package vfio
+
+import "golang.org/x/sys/unix"
+
+// Major is unix.Major
+func Major(dev uint64) uint32 {
+	return unix.Major(dev)
+}
+
+// Minor is unix.Minor
+func Minor(dev uint64) uint32 {
+	return unix.Minor(dev)
+}

From 44501ef8f6960a1e012aa53c4c8ca5019a6da3e8 Mon Sep 17 00:00:00 2001
From: Vladimir Popov <vladimir.popov@xored.com>
Date: Thu, 10 Dec 2020 09:42:46 +0700
Subject: [PATCH 11/11] Wait for driver to be bound

Signed-off-by: Vladimir Popov <vladimir.popov@xored.com>
---
 .../common/resourcepool/server.go             |  4 +-
 pkg/sriov/pci/pool.go                         | 83 ++++++++++++++++---
 2 files changed, 74 insertions(+), 13 deletions(-)

diff --git a/pkg/networkservice/common/resourcepool/server.go b/pkg/networkservice/common/resourcepool/server.go
index 6b211524..7e13f659 100644
--- a/pkg/networkservice/common/resourcepool/server.go
+++ b/pkg/networkservice/common/resourcepool/server.go
@@ -42,7 +42,7 @@ const (
 // PCIPool is a pci.Pool interface
 type PCIPool interface {
 	GetPCIFunction(pciAddr string) (sriov.PCIFunction, error)
-	BindDriver(iommuGroup uint, driverType sriov.DriverType) error
+	BindDriver(ctx context.Context, iommuGroup uint, driverType sriov.DriverType) error
 }
 
 // ResourcePool is a resource.Pool interface
@@ -103,7 +103,7 @@ func (s *resourcePoolServer) Request(ctx context.Context, request *networkservic
 			return errors.Wrapf(err, "failed to get VF IOMMU group: %v", vf.GetPCIAddress())
 		}
 
-		if err := s.pciPool.BindDriver(iommuGroup, s.driverType); err != nil {
+		if err := s.pciPool.BindDriver(ctx, iommuGroup, s.driverType); err != nil {
 			return err
 		}
 
diff --git a/pkg/sriov/pci/pool.go b/pkg/sriov/pci/pool.go
index 1da619d8..15caebe7 100644
--- a/pkg/sriov/pci/pool.go
+++ b/pkg/sriov/pci/pool.go
@@ -18,6 +18,12 @@
 package pci
 
 import (
+	"context"
+	"os"
+	"path/filepath"
+	"strconv"
+	"time"
+
 	"github.com/pkg/errors"
 
 	"github.com/networkservicemesh/sdk-sriov/pkg/sriov"
@@ -27,7 +33,9 @@ import (
 )
 
 const (
-	vfioDriver = "vfio-pci"
+	vfioDriver        = "vfio-pci"
+	driverBindTimeout = time.Second
+	driverBindCheck   = driverBindTimeout / 10
 )
 
 type pciFunction interface {
@@ -41,6 +49,8 @@ type pciFunction interface {
 type Pool struct {
 	functions             map[string]*function // pciAddr -> *function
 	functionsByIOMMUGroup map[uint][]*function // iommuGroup -> []*function
+	vfioDir               string
+	test                  bool
 }
 
 type function struct {
@@ -49,10 +59,11 @@ type function struct {
 }
 
 // NewPool returns a new PCI Pool
-func NewPool(pciDevicesPath, pciDriversPath string, cfg *config.Config) (*Pool, error) {
+func NewPool(pciDevicesPath, pciDriversPath, vfioDir string, cfg *config.Config) (*Pool, error) {
 	p := &Pool{
 		functions:             map[string]*function{},
 		functionsByIOMMUGroup: map[uint][]*function{},
+		vfioDir:               vfioDir,
 	}
 
 	for pfPCIAddr, pfCfg := range cfg.PhysicalFunctions {
@@ -80,6 +91,7 @@ func NewTestPool(physicalFunctions map[string]*sriovtest.PCIPhysicalFunction, cf
 	p := &Pool{
 		functions:             map[string]*function{},
 		functionsByIOMMUGroup: map[uint][]*function{},
+		test:                  true,
 	}
 
 	for pfPCIAddr, pfCfg := range cfg.PhysicalFunctions {
@@ -117,15 +129,15 @@ func (p *Pool) addFunction(pcif pciFunction, kernelDriver string) (err error) {
 
 // GetPCIFunction returns PCI function for the given PCI address
 func (p *Pool) GetPCIFunction(pciAddr string) (sriov.PCIFunction, error) {
-	f, err := p.find(pciAddr)
-	if err != nil {
-		return nil, err
+	f, ok := p.functions[pciAddr]
+	if !ok {
+		return nil, errors.Errorf("PCI function doesn't exist: %v", pciAddr)
 	}
 	return f.function, nil
 }
 
 // BindDriver binds selected IOMMU group to the given driver type
-func (p *Pool) BindDriver(iommuGroup uint, driverType sriov.DriverType) error {
+func (p *Pool) BindDriver(ctx context.Context, iommuGroup uint, driverType sriov.DriverType) error {
 	for _, f := range p.functionsByIOMMUGroup[iommuGroup] {
 		switch driverType {
 		case sriov.KernelDriver:
@@ -140,13 +152,62 @@ func (p *Pool) BindDriver(iommuGroup uint, driverType sriov.DriverType) error {
 			return errors.Errorf("driver type is not supported: %v", driverType)
 		}
 	}
+
+	for _, f := range p.functionsByIOMMUGroup[iommuGroup] {
+		if err := p.waitDriverGettingBound(ctx, f.function, driverType); err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 
-func (p *Pool) find(pciAddr string) (*function, error) {
-	f, ok := p.functions[pciAddr]
-	if !ok {
-		return nil, errors.Errorf("PCI function doesn't exist: %v", pciAddr)
+func (p *Pool) waitDriverGettingBound(ctx context.Context, pcif pciFunction, driverType sriov.DriverType) error {
+	timeoutCh := time.After(driverBindTimeout)
+	for {
+		var driverCheck func(pciFunction) error
+		switch driverType {
+		case sriov.KernelDriver:
+			driverCheck = p.kernelDriverCheck
+		case sriov.VFIOPCIDriver:
+			driverCheck = p.vfioDriverCheck
+		default:
+			return errors.Errorf("driver type is not supported: %v", driverType)
+		}
+
+		if driverCheck(pcif) == nil {
+			return nil
+		}
+
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-timeoutCh:
+			return errors.Errorf("time for binding kernel driver exceeded: %s", pcif.GetPCIAddress())
+		case <-time.After(driverBindCheck):
+		}
+	}
+}
+
+func (p *Pool) kernelDriverCheck(pcif pciFunction) error {
+	if p.test {
+		return nil
+	}
+
+	_, err := pcif.GetNetInterfaceName()
+	return err
+}
+
+func (p *Pool) vfioDriverCheck(pcif pciFunction) error {
+	if p.test {
+		return nil
 	}
-	return f, nil
+
+	iommuGroup, err := pcif.GetIOMMUGroup()
+	if err != nil {
+		return err
+	}
+
+	_, err = os.Stat(filepath.Join(p.vfioDir, strconv.FormatUint(uint64(iommuGroup), 10)))
+	return err
 }