Skip to content

Commit

Permalink
Merge pull request #377 from lmilleri/vdpa
Browse files Browse the repository at this point in the history 
Vdpa introduction
  • Loading branch information
@bn222
bn222 authored Mar 1, 2023
2 parents 331d1ae + f01bcd9 commit 13405c5
Show file tree
Hide file tree
Showing 28 changed files with 1,276 additions and 295 deletions.
1 change: 1 addition & 0 deletions api/v1/helper.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -379,6 +379,7 @@ func (p *SriovNetworkNodePolicy) generateVfGroup(iface *InterfaceExt) (*VfGroup,
PolicyName: p.GetName(),
Mtu: p.Spec.Mtu,
IsRdma: p.Spec.IsRdma,
VdpaType: p.Spec.VdpaType,
}, nil
}

Expand Down
125 changes: 98 additions & 27 deletions api/v1/helper_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,14 @@ import (
"bytes"
"encoding/json"
"flag"
"io/ioutil"
"os"
"path/filepath"
"testing"

"github.com/google/go-cmp/cmp"

v1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1"
"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

Expand Down Expand Up @@ -77,7 +78,30 @@ func newNodePolicy() *v1.SriovNetworkNodePolicy {
Name: "p1",
},
Spec: v1.SriovNetworkNodePolicySpec{
DeviceType: "netdevice",
DeviceType: consts.DeviceTypeNetDevice,
NicSelector: v1.SriovNetworkNicSelector{
PfNames: []string{"ens803f1"},
RootDevices: []string{"0000:86:00.1"},
Vendor: "8086",
},
NodeSelector: map[string]string{
"feature.node.kubernetes.io/network-sriov.capable": "true",
},
NumVfs: 2,
Priority: 99,
ResourceName: "p1res",
},
}
}

func newVdpaNodePolicy() *v1.SriovNetworkNodePolicy {
return &v1.SriovNetworkNodePolicy{
ObjectMeta: metav1.ObjectMeta{
Name: "p1",
},
Spec: v1.SriovNetworkNodePolicySpec{
DeviceType: consts.DeviceTypeNetDevice,
VdpaType: consts.VdpaTypeVirtio,
NicSelector: v1.SriovNetworkNicSelector{
PfNames: []string{"ens803f1"},
RootDevices: []string{"0000:86:00.1"},
Expand Down Expand Up @@ -138,11 +162,11 @@ func TestRendering(t *testing.T) {
gp := filepath.Join("testdata", filepath.FromSlash(t.Name())+".golden")
if *update {
t.Log("update golden file")
if err := ioutil.WriteFile(gp, b.Bytes(), 0644); err != nil {
if err := os.WriteFile(gp, b.Bytes(), 0644); err != nil {
t.Fatalf("failed to update golden file: %s", err)
}
}
g, err := ioutil.ReadFile(gp)
g, err := os.ReadFile(gp)
if err != nil {
t.Fatalf("failed reading .golden: %s", err)
}
Expand Down Expand Up @@ -185,11 +209,11 @@ func TestIBRendering(t *testing.T) {
gp := filepath.Join("testdata", filepath.FromSlash(t.Name())+".golden")
if *update {
t.Log("update golden file")
if err := ioutil.WriteFile(gp, b.Bytes(), 0644); err != nil {
if err := os.WriteFile(gp, b.Bytes(), 0644); err != nil {
t.Fatalf("failed to update golden file: %s", err)
}
}
g, err := ioutil.ReadFile(gp)
g, err := os.ReadFile(gp)
if err != nil {
t.Fatalf("failed reading .golden: %s", err)
}
Expand Down Expand Up @@ -222,7 +246,7 @@ func TestSriovNetworkNodePolicyApply(t *testing.T) {
PciAddress: "0000:86:00.1",
VfGroups: []v1.VfGroup{
{
DeviceType: "netdevice",
DeviceType: consts.DeviceTypeNetDevice,
ResourceName: "p1res",
VfRange: "0-1",
PolicyName: "p1",
Expand All @@ -242,7 +266,7 @@ func TestSriovNetworkNodePolicyApply(t *testing.T) {
PciAddress: "0000:86:00.0",
VfGroups: []v1.VfGroup{
{
DeviceType: "netdevice",
DeviceType: consts.DeviceTypeNetDevice,
ResourceName: "prevres",
VfRange: "0-1",
PolicyName: "p2",
Expand All @@ -261,7 +285,7 @@ func TestSriovNetworkNodePolicyApply(t *testing.T) {
PciAddress: "0000:86:00.0",
VfGroups: []v1.VfGroup{
{
DeviceType: "netdevice",
DeviceType: consts.DeviceTypeNetDevice,
ResourceName: "prevres",
VfRange: "0-1",
PolicyName: "p2",
Expand All @@ -274,7 +298,7 @@ func TestSriovNetworkNodePolicyApply(t *testing.T) {
PciAddress: "0000:86:00.1",
VfGroups: []v1.VfGroup{
{
DeviceType: "netdevice",
DeviceType: consts.DeviceTypeNetDevice,
ResourceName: "p1res",
VfRange: "0-1",
PolicyName: "p1",
Expand All @@ -296,7 +320,7 @@ func TestSriovNetworkNodePolicyApply(t *testing.T) {
PciAddress: "0000:86:00.1",
VfGroups: []v1.VfGroup{
{
DeviceType: "vfio-pci",
DeviceType: consts.DeviceTypeVfioPci,
ResourceName: "vfiores",
VfRange: "0-1",
PolicyName: "p2",
Expand All @@ -315,7 +339,7 @@ func TestSriovNetworkNodePolicyApply(t *testing.T) {
PciAddress: "0000:86:00.1",
VfGroups: []v1.VfGroup{
{
DeviceType: "netdevice",
DeviceType: consts.DeviceTypeNetDevice,
ResourceName: "p1res",
VfRange: "0-1",
PolicyName: "p1",
Expand All @@ -338,7 +362,7 @@ func TestSriovNetworkNodePolicyApply(t *testing.T) {
Mtu: 2000,
VfGroups: []v1.VfGroup{
{
DeviceType: "vfio-pci",
DeviceType: consts.DeviceTypeVfioPci,
ResourceName: "vfiores",
VfRange: "0-1",
PolicyName: "p2",
Expand All @@ -358,7 +382,7 @@ func TestSriovNetworkNodePolicyApply(t *testing.T) {
PciAddress: "0000:86:00.1",
VfGroups: []v1.VfGroup{
{
DeviceType: "netdevice",
DeviceType: consts.DeviceTypeNetDevice,
ResourceName: "p1res",
VfRange: "0-1",
PolicyName: "p1",
Expand All @@ -379,7 +403,7 @@ func TestSriovNetworkNodePolicyApply(t *testing.T) {
PciAddress: "0000:86:00.1",
VfGroups: []v1.VfGroup{
{
DeviceType: "vfio-pci",
DeviceType: consts.DeviceTypeVfioPci,
ResourceName: "vfiores",
VfRange: "2-4",
PolicyName: "p2",
Expand All @@ -398,13 +422,13 @@ func TestSriovNetworkNodePolicyApply(t *testing.T) {
PciAddress: "0000:86:00.1",
VfGroups: []v1.VfGroup{
{
DeviceType: "netdevice",
DeviceType: consts.DeviceTypeNetDevice,
ResourceName: "p1res",
VfRange: "0-1",
PolicyName: "p1",
},
{
DeviceType: "vfio-pci",
DeviceType: consts.DeviceTypeVfioPci,
ResourceName: "vfiores",
VfRange: "2-4",
PolicyName: "p2",
Expand All @@ -426,13 +450,13 @@ func TestSriovNetworkNodePolicyApply(t *testing.T) {
PciAddress: "0000:86:00.1",
VfGroups: []v1.VfGroup{
{
DeviceType: "vfio-pci",
DeviceType: consts.DeviceTypeVfioPci,
ResourceName: "vfiores1",
VfRange: "0-0",
PolicyName: "p2",
},
{
DeviceType: "vfio-pci",
DeviceType: consts.DeviceTypeVfioPci,
ResourceName: "vfiores2",
VfRange: "1-1",
PolicyName: "p3",
Expand All @@ -451,7 +475,7 @@ func TestSriovNetworkNodePolicyApply(t *testing.T) {
PciAddress: "0000:86:00.1",
VfGroups: []v1.VfGroup{
{
DeviceType: "netdevice",
DeviceType: consts.DeviceTypeNetDevice,
ResourceName: "p1res",
VfRange: "0-1",
PolicyName: "p1",
Expand All @@ -473,7 +497,7 @@ func TestSriovNetworkNodePolicyApply(t *testing.T) {
PciAddress: "0000:86:00.1",
VfGroups: []v1.VfGroup{
{
DeviceType: "vfio-pci",
DeviceType: consts.DeviceTypeVfioPci,
ResourceName: "p1res",
VfRange: "2-3",
PolicyName: "p2",
Expand All @@ -492,7 +516,7 @@ func TestSriovNetworkNodePolicyApply(t *testing.T) {
PciAddress: "0000:86:00.1",
VfGroups: []v1.VfGroup{
{
DeviceType: "netdevice",
DeviceType: consts.DeviceTypeNetDevice,
ResourceName: "p1res",
VfRange: "0-1",
PolicyName: "p1",
Expand All @@ -514,7 +538,7 @@ func TestSriovNetworkNodePolicyApply(t *testing.T) {
PciAddress: "0000:86:00.1",
VfGroups: []v1.VfGroup{
{
DeviceType: "vfio-pci",
DeviceType: consts.DeviceTypeVfioPci,
ResourceName: "p2res",
VfRange: "2-3",
PolicyName: "p2",
Expand All @@ -533,13 +557,13 @@ func TestSriovNetworkNodePolicyApply(t *testing.T) {
PciAddress: "0000:86:00.1",
VfGroups: []v1.VfGroup{
{
DeviceType: "netdevice",
DeviceType: consts.DeviceTypeNetDevice,
ResourceName: "p1res",
VfRange: "0-1",
PolicyName: "p1",
},
{
DeviceType: "vfio-pci",
DeviceType: consts.DeviceTypeVfioPci,
ResourceName: "p2res",
VfRange: "2-3",
PolicyName: "p2",
Expand All @@ -556,7 +580,7 @@ func TestSriovNetworkNodePolicyApply(t *testing.T) {
Name: "p1",
},
Spec: v1.SriovNetworkNodePolicySpec{
DeviceType: "netdevice",
DeviceType: consts.DeviceTypeNetDevice,
NicSelector: v1.SriovNetworkNicSelector{
PfNames: []string{},
RootDevices: []string{},
Expand All @@ -580,7 +604,7 @@ func TestSriovNetworkNodePolicyApply(t *testing.T) {
Name: "p1",
},
Spec: v1.SriovNetworkNodePolicySpec{
DeviceType: "netdevice",
DeviceType: consts.DeviceTypeNetDevice,
NicSelector: v1.SriovNetworkNicSelector{
PfNames: []string{"ens803f0#a-c"},
RootDevices: []string{},
Expand Down Expand Up @@ -612,3 +636,50 @@ func TestSriovNetworkNodePolicyApply(t *testing.T) {
})
}
}

func TestVdpaNodePolicyApply(t *testing.T) {
testtable := []struct {
tname string
currentState *v1.SriovNetworkNodeState
policy *v1.SriovNetworkNodePolicy
expectedInterfaces v1.Interfaces
equalP bool
expectedErr bool
}{
{
tname: "vdpa configuration",
currentState: newNodeState(),
policy: newVdpaNodePolicy(),
equalP: false,
expectedInterfaces: []v1.Interface{
{
Name: "ens803f1",
NumVfs: 2,
PciAddress: "0000:86:00.1",
VfGroups: []v1.VfGroup{
{
DeviceType: consts.DeviceTypeNetDevice,
VdpaType: consts.VdpaTypeVirtio,
ResourceName: "p1res",
VfRange: "0-1",
PolicyName: "p1",
},
},
},
},
},
}
for _, tc := range testtable {
t.Run(tc.tname, func(t *testing.T) {
err := tc.policy.Apply(tc.currentState, tc.equalP)
if tc.expectedErr && err == nil {
t.Errorf("Apply expecting error.")
} else if !tc.expectedErr && err != nil {
t.Errorf("Apply error:\n%s", err)
}
if diff := cmp.Diff(tc.expectedInterfaces, tc.currentState.Spec.Interfaces); diff != "" {
t.Errorf("SriovNetworkNodeState spec diff (-want +got):\n%s", diff)
}
})
}
}
3 changes: 3 additions & 0 deletions api/v1/sriovnetworknodepolicy_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,9 @@ type SriovNetworkNodePolicySpec struct {
// +kubebuilder:validation:Enum=legacy;switchdev
// NIC Device Mode. Allowed value "legacy","switchdev".
EswitchMode string `json:"eSwitchMode,omitempty"`
// +kubebuilder:validation:Enum=virtio
// VDPA device type. Allowed value "virtio"
VdpaType string `json:"vdpaType,omitempty"`
}

type SriovNetworkNicSelector struct {
Expand Down
1 change: 1 addition & 0 deletions api/v1/sriovnetworknodestate_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ type VfGroup struct {
PolicyName string `json:"policyName,omitempty"`
Mtu int `json:"mtu,omitempty"`
IsRdma bool `json:"isRdma,omitempty"`
VdpaType string `json:"vdpaType,omitempty"`
}

type Interfaces []Interface
Expand Down
35 changes: 34 additions & 1 deletion bindata/manifests/switchdev-config/files/switchdev-configuration-after-nm.sh.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,24 @@ contents:
set -eux
input="/etc/sriov_config.json"
minId=-1
maxId=-1
extract_min_max_ids() {
range=$(jq -c '.vfRange' -r <<< $group)
ids=(${range//-/ })
length=${#ids[@]}
minId=-1
maxId=-1
if [[ length -eq 2 ]]; then
minId=${ids[0]}
maxId=${ids[1]}
elif [[ length -eq 1 ]]; then
minId=${ids[0]}
maxId=$minId
fi
}
if [ ! -f $input ]; then
echo "File /etc/sriov_config.json not exist."
exit
Expand All @@ -25,11 +43,26 @@ contents:
echo "load VF driver for $pci_addr"
VfDirs=$(ls /sys/bus/pci/devices/${pci_addr} | grep virtfn)
# load VF driver
# load VF driver and configure vdpa if needed
for VfDir in $VfDirs
do
VfPciAddr=$(basename "$( readlink -f /sys/bus/pci/devices/${pci_addr}/$VfDir )")
echo $VfPciAddr > /sys/bus/pci/drivers_probe
# extract VF id from a path like '/sys/bus/pci/devices/0000:65:00.0/virtfn1'
# the VF id is the 7th character after 'virtfn'
vfid=${VfDir:6}
# check if vfid in VF group range
jq -c '.vfGroups[]' <<< "$iface" | while read group;
do
extract_min_max_ids
vdpaType=$(jq -c '.vdpaType' -r <<< $group)
if [ $vfid -le $maxId ] && [ $vfid -ge $minId ] && [ $vdpaType == "virtio" ]; then
vdpa_cmd="vdpa dev add name vdpa:"${VfPciAddr}" mgmtdev pci/"${VfPciAddr}
eval $vdpa_cmd
fi
done
done
fi
done
Loading

0 comments on commit 13405c5

Please sign in to comment.