incusd/device/gpu_sriov: Implement NUMA fallback (NPS)

internal/server/device/gpu_sriov.go

@@ -126,12 +126,60 @@ func (d *gpuSRIOV) getVF() (string, int, error) {
 
 	// If NUMA restricted, build up a list of nodes.
 	var numaNodeSet []int64
+	var numaNodeSetFallback []int64
 	if d.inst.ExpandedConfig()["limits.cpu.nodes"] != "" {
 		// Parse the NUMA restriction.
 		numaNodeSet, err = resources.ParseNumaNodeSet(d.inst.ExpandedConfig()["limits.cpu.nodes"])
 		if err != nil {
 			return "", -1, err
 		}
+
+		// List all the CPUs.
+		cpus, err := resources.GetCPU()
+		if err != nil {
+			return "", -1, err
+		}
+
+		// Get list of socket IDs from the list of NUMA nodes.
+		numaSockets := make([]uint64, 0, len(cpus.Sockets))
+
+		for _, cpuSocket := range cpus.Sockets {
+			if slices.Contains(numaSockets, cpuSocket.Socket) {
+				continue
+			}
+
+			for _, cpuCore := range cpuSocket.Cores {
+				found := false
+				for _, cpuThread := range cpuCore.Threads {
+					if slices.Contains(numaNodeSet, int64(cpuThread.NUMANode)) {
+						numaSockets = append(numaSockets, cpuSocket.Socket)
+						found = true
+						break
+					}
+				}
+
+				if found {
+					break
+				}
+			}
+		}
+
+		// Get the list of NUMA nodes from the socket list.
+		numaNodeSetFallback = []int64{}
+
+		for _, cpuSocket := range cpus.Sockets {
+			if !slices.Contains(numaSockets, cpuSocket.Socket) {
+				continue
+			}
+
+			for _, cpuCore := range cpuSocket.Cores {
+				for _, cpuThread := range cpuCore.Threads {
+					if !slices.Contains(numaNodeSetFallback, int64(cpuThread.NUMANode)) {
+						numaNodeSetFallback = append(numaNodeSetFallback, int64(cpuThread.NUMANode))
+					}
+				}
+			}
+		}
 	}
 
 	// Locate a suitable VF from the least loaded suitable card.
@@ -177,6 +225,17 @@ func (d *gpuSRIOV) getVF() (string, int, error) {
 			continue
 		}
 
+		// If NUMA node restrictions are in place, continue with fallback set.
+		if numaNodeSetFallback != nil && !slices.Contains(numaNodeSet, int64(cardNUMA)) && !slices.Contains(numaNodeSetFallback, int64(cardNUMA)) && slices.Contains(numaNodeSetFallback, int64(gpu.NUMANode)) {
+			pciAddress = gpu.PCIAddress
+			vfID = vfs[0]
+			cardAvailable = len(vfs)
+			cardTotal = int(gpu.SRIOV.CurrentVFs)
+			cardNUMA = int(gpu.NUMANode)
+
+			continue
+		}
+
 		// Prioritize less busy cards.
 		if pciAddress == "" || (float64(len(vfs))/float64(gpu.SRIOV.CurrentVFs)) > (float64(cardAvailable)/float64(cardTotal)) {
 			pciAddress = gpu.PCIAddress