diff --git a/docs/storage/prometheus.md b/docs/storage/prometheus.md index 94eb993fec..7a005bc97a 100644 --- a/docs/storage/prometheus.md +++ b/docs/storage/prometheus.md @@ -114,6 +114,7 @@ Metric name | Type | Description | Unit (where applicable) | option parameter | `machine_dimm_capacity_bytes` | Gauge | Total RAM DIMM capacity (all types memory modules) value labeled by dimm type,
information is retrieved from sysfs edac per-DIMM API (/sys/devices/system/edac/mc/) introduced in kernel 3.6 | bytes | | | `machine_dimm_count` | Gauge | Number of RAM DIMM (all types memory modules) value labeled by dimm type,
information is retrieved from sysfs edac per-DIMM API (/sys/devices/system/edac/mc/) introduced in kernel 3.6 | | | `machine_memory_bytes` | Gauge | Amount of memory installed on the machine | bytes | | +`machine_node_distance` | Gauge | Distance between NUMA node and target NUMA node | | cpu_topology | `machine_node_hugepages_count` | Gauge | Numer of hugepages assigned to NUMA node | | cpu_topology | `machine_node_memory_capacity_bytes` | Gauge | Amount of memory assigned to NUMA node | bytes | cpu_topology | `machine_nvm_avg_power_budget_watts` | Gauge | NVM power budget | watts | | libipmctl diff --git a/info/v1/machine.go b/info/v1/machine.go index 108089bc22..95d8b9ac4c 100644 --- a/info/v1/machine.go +++ b/info/v1/machine.go @@ -44,6 +44,7 @@ type Node struct { HugePages []HugePagesInfo `json:"hugepages"` Cores []Core `json:"cores"` Caches []Cache `json:"caches"` + Distances []uint64 `json:"distances"` } type Core struct { diff --git a/machine/topology_test.go b/machine/topology_test.go index 0cc81b9225..b1d03b9dd6 100644 --- a/machine/topology_test.go +++ b/machine/topology_test.go @@ -204,6 +204,10 @@ func TestTopology(t *testing.T) { "/fakeSysfs/devices/system/node/node0/cpu11": "1", } sysFs.SetPhysicalPackageIDs(physicalPackageIDs, nil) + + sysFs.SetDistances("/fakeSysfs/devices/system/node/node0", "10 11", nil) + sysFs.SetDistances("/fakeSysfs/devices/system/node/node1", "11 10", nil) + topology, numCores, err := GetTopology(sysFs) assert.Nil(t, err) assert.Equal(t, 12, numCores) @@ -217,12 +221,17 @@ func TestTopology(t *testing.T) { Type: "unified", Level: 1, } + distances := [][]uint64{ + {10, 11}, + {11, 10}, + } for i := 0; i < numNodes; i++ { node := info.Node{Id: i} // Copy over Memory from result. TODO(rjnagal): Use memory from fake. node.Memory = topology[i].Memory // Copy over HugePagesInfo from result. TODO(ohsewon): Use HugePagesInfo from fake. node.HugePages = topology[i].HugePages + node.Distances = distances[i] for j := 0; j < numCoresPerNode; j++ { core := info.Core{Id: i*numCoresPerNode + j} core.Caches = append(core.Caches, cache) @@ -298,12 +307,13 @@ func TestTopologyWithoutNodes(t *testing.T) { topologyJSON2, err := json.Marshal(topology[1]) assert.Nil(t, err) - expectedTopology1 := `{"node_id":0,"memory":0,"hugepages":null,"cores":[{"core_id":0,"thread_ids":[0,2],"caches":[{"id":0, "size":32768,"type":"unified","level":0}], "socket_id": 0, "uncore_caches":null}],"caches":null}` + expectedTopology1 := `{"node_id":0,"memory":0,"hugepages":null,"distances":null,"cores":[{"core_id":0,"thread_ids":[0,2],"caches":[{"id":0, "size":32768,"type":"unified","level":0}], "socket_id": 0, "uncore_caches":null}],"caches":null}` expectedTopology2 := ` { "node_id":1, "memory":0, "hugepages":null, + "distances": null, "cores":[ { "core_id":1, @@ -359,6 +369,9 @@ func TestTopologyWithNodesWithoutCPU(t *testing.T) { } sysFs.SetHugePagesNr(hugePageNr, nil) + sysFs.SetDistances("/fakeSysfs/devices/system/node/node0", "10 11", nil) + sysFs.SetDistances("/fakeSysfs/devices/system/node/node1", "11 10", nil) + topology, numCores, err := GetTopology(sysFs) assert.Nil(t, err) @@ -381,6 +394,10 @@ func TestTopologyWithNodesWithoutCPU(t *testing.T) { "page_size": 1048576 } ], + "distances": [ + 10, + 11 + ], "memory": 33604804608, "node_id": 0 }, @@ -397,6 +414,10 @@ func TestTopologyWithNodesWithoutCPU(t *testing.T) { "page_size": 1048576 } ], + "distances": [ + 11, + 10 + ], "memory": 33604804608, "node_id": 1 } diff --git a/metrics/prometheus_fake.go b/metrics/prometheus_fake.go index 822b3f82c9..df45e72076 100644 --- a/metrics/prometheus_fake.go +++ b/metrics/prometheus_fake.go @@ -153,6 +153,10 @@ func (p testSubcontainersInfoProvider) GetMachineInfo() (*info.MachineInfo, erro }, }, }, + Distances: []uint64{ + 10, + 12, + }, }, { Id: 1, @@ -260,6 +264,10 @@ func (p testSubcontainersInfoProvider) GetMachineInfo() (*info.MachineInfo, erro Level: 3, }, }, + Distances: []uint64{ + 12, + 10, + }, }, }, }, nil diff --git a/metrics/prometheus_machine.go b/metrics/prometheus_machine.go index f60cdc2883..181aaf81b3 100644 --- a/metrics/prometheus_machine.go +++ b/metrics/prometheus_machine.go @@ -28,13 +28,14 @@ import ( var baseLabelsNames = []string{"machine_id", "system_uuid", "boot_id"} const ( - prometheusModeLabelName = "mode" - prometheusTypeLabelName = "type" - prometheusLevelLabelName = "level" - prometheusNodeLabelName = "node_id" - prometheusCoreLabelName = "core_id" - prometheusThreadLabelName = "thread_id" - prometheusPageSizeLabelName = "page_size" + prometheusModeLabelName = "mode" + prometheusTypeLabelName = "type" + prometheusLevelLabelName = "level" + prometheusNodeLabelName = "node_id" + prometheusCoreLabelName = "core_id" + prometheusThreadLabelName = "thread_id" + prometheusPageSizeLabelName = "page_size" + prometheusTargetNodeLabelName = "target_node_id" nvmMemoryMode = "memory_mode" nvmAppDirectMode = "app_direct_mode" @@ -191,6 +192,15 @@ func NewPrometheusMachineCollector(i infoProvider, includedMetrics container.Met return getHugePagesCount(machineInfo) }, }, + { + name: "machine_node_distance", + help: "Distance between NUMA node and target NUMA node.", + valueType: prometheus.GaugeValue, + extraLabels: []string{prometheusNodeLabelName, prometheusTargetNodeLabelName}, + getValues: func(machineInfo *info.MachineInfo) metricValues { + return getDistance(machineInfo) + }, + }, }...) } return c @@ -356,3 +366,19 @@ func getCaches(machineInfo *info.MachineInfo) metricValues { } return mValues } + +func getDistance(machineInfo *info.MachineInfo) metricValues { + mValues := make(metricValues, 0, len(machineInfo.Topology)^2) + for _, node := range machineInfo.Topology { + nodeID := strconv.Itoa(node.Id) + for i, target := range node.Distances { + mValues = append(mValues, + metricValue{ + value: float64(target), + labels: []string{nodeID, strconv.Itoa(i)}, + timestamp: machineInfo.Timestamp, + }) + } + } + return mValues +} diff --git a/metrics/prometheus_machine_test.go b/metrics/prometheus_machine_test.go index 8f97cf7df2..e2d8d555ac 100644 --- a/metrics/prometheus_machine_test.go +++ b/metrics/prometheus_machine_test.go @@ -188,6 +188,22 @@ func TestGetHugePagesCount(t *testing.T) { assertMetricValues(t, expectedMetricVals, metricVals, "Unexpected information about Node memory") } +func TestGetDistance(t *testing.T) { + machineInfo, err := testSubcontainersInfoProvider{}.GetMachineInfo() + assert.Nil(t, err) + + metricVals := getDistance(machineInfo) + + assert.Equal(t, 4, len(metricVals)) + expectedMetricVals := []metricValue{ + {value: 10, labels: []string{"0", "0"}, timestamp: time.Unix(1395066363, 0)}, + {value: 12, labels: []string{"0", "1"}, timestamp: time.Unix(1395066363, 0)}, + {value: 12, labels: []string{"1", "0"}, timestamp: time.Unix(1395066363, 0)}, + {value: 10, labels: []string{"1", "1"}, timestamp: time.Unix(1395066363, 0)}, + } + assertMetricValues(t, expectedMetricVals, metricVals, "Unexpected information about Node memory") +} + func assertMetricValues(t *testing.T, expected metricValues, actual metricValues, message string) { for i := range actual { assert.Truef(t, reflect.DeepEqual(expected[i], actual[i]), diff --git a/metrics/testdata/prometheus_machine_metrics b/metrics/testdata/prometheus_machine_metrics index 1760427885..c872448318 100644 --- a/metrics/testdata/prometheus_machine_metrics +++ b/metrics/testdata/prometheus_machine_metrics @@ -45,6 +45,12 @@ machine_dimm_count{boot_id="boot-id-test",machine_id="machine-id-test",system_uu # HELP machine_memory_bytes Amount of memory installed on the machine. # TYPE machine_memory_bytes gauge machine_memory_bytes{boot_id="boot-id-test",machine_id="machine-id-test",system_uuid="system-uuid-test"} 1024 1395066363000 +# HELP machine_node_distance Distance between NUMA node and target NUMA node. +# TYPE machine_node_distance gauge +machine_node_distance{boot_id="boot-id-test",machine_id="machine-id-test",node_id="0",system_uuid="system-uuid-test",target_node_id="0"} 10 1395066363000 +machine_node_distance{boot_id="boot-id-test",machine_id="machine-id-test",node_id="0",system_uuid="system-uuid-test",target_node_id="1"} 12 1395066363000 +machine_node_distance{boot_id="boot-id-test",machine_id="machine-id-test",node_id="1",system_uuid="system-uuid-test",target_node_id="0"} 12 1395066363000 +machine_node_distance{boot_id="boot-id-test",machine_id="machine-id-test",node_id="1",system_uuid="system-uuid-test",target_node_id="1"} 10 1395066363000 # HELP machine_node_hugepages_count Numer of hugepages assigned to NUMA node. # TYPE machine_node_hugepages_count gauge machine_node_hugepages_count{boot_id="boot-id-test",machine_id="machine-id-test",node_id="0",page_size="1048576",system_uuid="system-uuid-test"} 0 1395066363000 diff --git a/utils/sysfs/fakesysfs/fake.go b/utils/sysfs/fakesysfs/fake.go index 6c9da743b2..5eac950aab 100644 --- a/utils/sysfs/fakesysfs/fake.go +++ b/utils/sysfs/fakesysfs/fake.go @@ -76,6 +76,9 @@ type FakeSysFs struct { hugePagesNr map[string]string hugePagesNrErr error + distances map[string]string + distancesErr error + onlineCPUs map[string]interface{} } @@ -201,6 +204,27 @@ func (fs *FakeSysFs) GetSystemUUID() (string, error) { return "1F862619-BA9F-4526-8F85-ECEAF0C97430", nil } +func (fs *FakeSysFs) GetDistances(nodeDir string) (string, error) { + if fs.distancesErr != nil { + return "", fs.distancesErr + } + + if _, ok := fs.distances[nodeDir]; !ok { + return "", fmt.Errorf("distance not found") + } + + return fs.distances[nodeDir], nil +} + +func (fs *FakeSysFs) SetDistances(nodeDir string, distances string, err error) { + if fs.distances == nil { + fs.distances = map[string]string{nodeDir: distances} + } else { + fs.distances[nodeDir] = distances + } + fs.distancesErr = err +} + func (fs *FakeSysFs) IsCPUOnline(dir string) bool { if fs.onlineCPUs == nil { return true diff --git a/utils/sysfs/sysfs.go b/utils/sysfs/sysfs.go index 580ef91eb9..0b83dd7500 100644 --- a/utils/sysfs/sysfs.go +++ b/utils/sysfs/sysfs.go @@ -38,6 +38,8 @@ const ( meminfoFile = "meminfo" + distanceFile = "distance" + sysFsCPUTopology = "topology" // CPUPhysicalPackageID is a physical package id of cpu#. Typically corresponds to a physical socket number, @@ -113,6 +115,10 @@ type SysFs interface { GetCacheInfo(cpu int, cache string) (CacheInfo, error) GetSystemUUID() (string, error) + + // GetDistances returns distance array + GetDistances(string) (string, error) + // IsCPUOnline determines if CPU status from kernel hotplug machanism standpoint. // See: https://www.kernel.org/doc/html/latest/core-api/cpu_hotplug.html IsCPUOnline(dir string) bool @@ -161,6 +167,15 @@ func (fs *realSysFs) GetMemInfo(nodePath string) (string, error) { return strings.TrimSpace(string(meminfo)), err } +func (fs *realSysFs) GetDistances(nodePath string) (string, error) { + distancePath := fmt.Sprintf("%s/%s", nodePath, distanceFile) + distance, err := ioutil.ReadFile(distancePath) + if err != nil { + return "", err + } + return strings.TrimSpace(string(distance)), err +} + func (fs *realSysFs) GetHugePagesInfo(hugePagesDirectory string) ([]os.FileInfo, error) { return ioutil.ReadDir(hugePagesDirectory) } diff --git a/utils/sysfs/sysfs_test.go b/utils/sysfs/sysfs_test.go index 5369f9995d..91f7dca1bd 100644 --- a/utils/sysfs/sysfs_test.go +++ b/utils/sysfs/sysfs_test.go @@ -305,3 +305,17 @@ func TestUniqueCPUPropertyOnSingleSocketMultipleNUMAsSystem(t *testing.T) { count = GetUniqueCPUPropertyCount("./testdata_single_socket_many_NUMAs/", CPUCoreID) assert.Equal(t, 16, count) } + +func TestGetDistances(t *testing.T) { + sysFs := NewRealSysFs() + distances, err := sysFs.GetDistances("./testdata/node0") + assert.Nil(t, err) + assert.Equal(t, "10 11", distances) +} + +func TestGetDistancesFileIsMissing(t *testing.T) { + sysFs := NewRealSysFs() + distances, err := sysFs.GetDistances("./testdata/node1") + assert.NotNil(t, err) + assert.Equal(t, "", distances) +} diff --git a/utils/sysfs/testdata/node0/distance b/utils/sysfs/testdata/node0/distance new file mode 100644 index 0000000000..13a954d904 --- /dev/null +++ b/utils/sysfs/testdata/node0/distance @@ -0,0 +1 @@ +10 11 diff --git a/utils/sysinfo/sysinfo.go b/utils/sysinfo/sysinfo.go index 47fea630cb..57b3653c12 100644 --- a/utils/sysinfo/sysinfo.go +++ b/utils/sysinfo/sysinfo.go @@ -243,6 +243,11 @@ func GetNodesInfo(sysFs sysfs.SysFs) ([]info.Node, int, error) { return nil, 0, err } + node.Distances, err = getDistances(sysFs, nodeDir) + if err != nil { + return nil, 0, err + } + nodes = append(nodes, node) } return nodes, allLogicalCoresCount, err @@ -391,6 +396,27 @@ func getNodeMemInfo(sysFs sysfs.SysFs, nodeDir string) (uint64, error) { return uint64(memory), nil } +// getDistances returns information about distances between NUMA nodes +func getDistances(sysFs sysfs.SysFs, nodeDir string) ([]uint64, error) { + rawDistance, err := sysFs.GetDistances(nodeDir) + if err != nil { + //Ignore if per-node info is not available. + klog.Warningf("Found node without distance information, nodeDir: %s", nodeDir) + return nil, nil + } + + distances := []uint64{} + for _, distance := range strings.Split(rawDistance, " ") { + distanceUint, err := strconv.ParseUint(distance, 10, 64) + if err != nil { + return nil, fmt.Errorf("cannot convert %s to int", distance) + } + distances = append(distances, distanceUint) + } + + return distances, nil +} + // getCoresInfo returns information about physical cores func getCoresInfo(sysFs sysfs.SysFs, cpuDirs []string) ([]info.Core, error) { cores := make([]info.Core, 0, len(cpuDirs)) diff --git a/utils/sysinfo/sysinfo_test.go b/utils/sysinfo/sysinfo_test.go index 17fbb92a31..4f424b80be 100644 --- a/utils/sysinfo/sysinfo_test.go +++ b/utils/sysinfo/sysinfo_test.go @@ -116,6 +116,7 @@ func TestGetNodesInfo(t *testing.T) { physicalPackageIDs map[string]string nodes int cores int + distances []string expectedNodes string }{ { @@ -161,6 +162,10 @@ func TestGetNodesInfo(t *testing.T) { }, 2, 4, + []string{ + "10 11", + "11 10", + }, ` [ { @@ -172,6 +177,10 @@ func TestGetNodesInfo(t *testing.T) { "num_pages": 1 } ], + "distances": [ + 10, + 11 + ], "cores": [ { "core_id": 0, @@ -202,6 +211,10 @@ func TestGetNodesInfo(t *testing.T) { "num_pages": 1 } ], + "distances": [ + 11, + 10 + ], "cores": [ { "core_id": 1, @@ -271,11 +284,17 @@ func TestGetNodesInfo(t *testing.T) { }, 1, 6, + []string{ + "10", + }, ` [ { "node_id": 0, "memory": 33604804608, + "distances": [ + 10 + ], "hugepages": [ { "page_size": 2048, @@ -338,6 +357,10 @@ func TestGetNodesInfo(t *testing.T) { fakeSys.SetHugePages(test.hugePages, nil) fakeSys.SetHugePagesNr(test.hugePageNr, nil) fakeSys.SetPhysicalPackageIDs(test.physicalPackageIDs, nil) + for i, node := range test.nodesPaths { + fakeSys.SetDistances(node, test.distances[i], nil) + } + nodes, cores, err := GetNodesInfo(fakeSys) assert.Nil(t, err) assert.Equal(t, test.nodes, len(nodes)) @@ -412,6 +435,9 @@ func TestGetNodesInfoWithOfflineCPUs(t *testing.T) { } fakeSys.SetPhysicalPackageIDs(physicalPackageIDs, nil) + fakeSys.SetDistances("/fakeSysfs/devices/system/node/node0", "10 11", nil) + fakeSys.SetDistances("/fakeSysfs/devices/system/node/node1", "11 10", nil) + nodes, cores, err := GetNodesInfo(fakeSys) assert.Nil(t, err) assert.Equal(t, 2, len(nodes)) @@ -430,6 +456,10 @@ func TestGetNodesInfoWithOfflineCPUs(t *testing.T) { "num_pages": 1 } ], + "distances": [ + 10, + 11 + ], "cores": [ { "core_id": 0, @@ -459,6 +489,10 @@ func TestGetNodesInfoWithOfflineCPUs(t *testing.T) { "num_pages": 1 } ], + "distances": [ + 11, + 10 + ], "cores": [ { "core_id": 1, @@ -588,6 +622,9 @@ func TestGetNodesInfoWithoutCacheInfo(t *testing.T) { } fakeSys.SetPhysicalPackageIDs(physicalPackageIDs, nil) + fakeSys.SetDistances("/fakeSysfs/devices/system/node/node0", "10 11", nil) + fakeSys.SetDistances("/fakeSysfs/devices/system/node/node1", "11 10", nil) + nodes, cores, err := GetNodesInfo(fakeSys) assert.Nil(t, err) assert.Equal(t, 2, len(nodes)) @@ -606,6 +643,10 @@ func TestGetNodesInfoWithoutCacheInfo(t *testing.T) { "num_pages": 1 } ], + "distances": [ + 10, + 11 + ], "cores": [ { "core_id": 0, @@ -629,6 +670,10 @@ func TestGetNodesInfoWithoutCacheInfo(t *testing.T) { "num_pages": 1 } ], + "distances": [ + 11, + 10 + ], "cores": [ { "core_id": 1, @@ -695,6 +740,9 @@ func TestGetNodesInfoWithoutHugePagesInfo(t *testing.T) { } fakeSys.SetPhysicalPackageIDs(physicalPackageIDs, nil) + fakeSys.SetDistances("/fakeSysfs/devices/system/node/node0", "10 11", nil) + fakeSys.SetDistances("/fakeSysfs/devices/system/node/node1", "11 10", nil) + nodes, cores, err := GetNodesInfo(fakeSys) assert.Nil(t, err) assert.Equal(t, 2, len(nodes)) @@ -707,6 +755,10 @@ func TestGetNodesInfoWithoutHugePagesInfo(t *testing.T) { { "node_id": 0, "memory": 33604804608, + "distances": [ + 10, + 11 + ], "hugepages": null, "cores": [ { @@ -733,6 +785,10 @@ func TestGetNodesInfoWithoutHugePagesInfo(t *testing.T) { "node_id": 1, "memory": 33604804608, "hugepages": null, + "distances": [ + 11, + 10 + ], "cores": [ { "core_id": 1, @@ -815,6 +871,7 @@ func TestGetNodesInfoWithoutNodes(t *testing.T) { { "node_id":0, "memory":0, + "distances": null, "hugepages":null, "cores":[ { @@ -840,6 +897,7 @@ func TestGetNodesInfoWithoutNodes(t *testing.T) { { "node_id":1, "memory":0, + "distances": null, "hugepages":null, "cores":[ { @@ -920,6 +978,7 @@ func TestGetNodesInfoWithoutNodesWhenPhysicalPackageIDMissingForOneCPU(t *testin { "node_id":0, "memory":0, + "distances": null, "hugepages":null, "cores":[ { @@ -1037,6 +1096,8 @@ func TestGetNodesWhenTopologyDirMissingForOneCPU(t *testing.T) { fakeSys.SetPhysicalPackageIDs(physicalPackageIDs, physicalPackageIDErrors) + fakeSys.SetDistances("/fakeSysfs/devices/system/node/node0", "10", nil) + nodes, cores, err := GetNodesInfo(fakeSys) assert.Nil(t, err) @@ -1054,6 +1115,9 @@ func TestGetNodesWhenTopologyDirMissingForOneCPU(t *testing.T) { { "node_id":0, "memory":33604804608, + "distances" : [ + 10 + ], "hugepages":[ { "page_size":2048, @@ -1129,6 +1193,8 @@ func TestGetNodesWhenPhysicalPackageIDMissingForOneCPU(t *testing.T) { fakeSys.SetPhysicalPackageIDs(physicalPackageIDs, physicalPackageIDErrors) + fakeSys.SetDistances("/fakeSysfs/devices/system/node/node0", "10", nil) + nodes, cores, err := GetNodesInfo(fakeSys) assert.Nil(t, err) @@ -1146,6 +1212,9 @@ func TestGetNodesWhenPhysicalPackageIDMissingForOneCPU(t *testing.T) { { "node_id":0, "memory":33604804608, + "distances" : [ + 10 + ], "hugepages":[ { "page_size":2048, @@ -1520,8 +1589,12 @@ func TestGetNodesInfoWithUncoreCacheInfo(t *testing.T) { } fakeSys.SetPhysicalPackageIDs(physicalPackageIDs, nil) + fakeSys.SetDistances("/fakeSysfs/devices/system/node/node0", "10 11", nil) + fakeSys.SetDistances("/fakeSysfs/devices/system/node/node1", "11 10", nil) + nodes, cores, err := GetNodesInfo(fakeSys) assert.Nil(t, err) + fmt.Println(err) assert.Equal(t, 2, len(nodes)) assert.Equal(t, 4, cores) @@ -1532,6 +1605,10 @@ func TestGetNodesInfoWithUncoreCacheInfo(t *testing.T) { { "node_id": 0, "memory": 33604804608, + "distances" : [ + 10, + 11 + ], "hugepages": null, "cores": [ { @@ -1557,6 +1634,10 @@ func TestGetNodesInfoWithUncoreCacheInfo(t *testing.T) { { "node_id": 1, "memory": 33604804608, + "distances" : [ + 11, + 10 + ], "hugepages": null, "cores": [ { @@ -1582,3 +1663,25 @@ func TestGetNodesInfoWithUncoreCacheInfo(t *testing.T) { ]` assert.JSONEq(t, expectedNodes, string(nodesJSON)) } + +func TestGetDistances(t *testing.T) { + fakeSys := &fakesysfs.FakeSysFs{} + node := "/fakeSysfs/devices/system/node/node0" + fakeSys.SetDistances(node, "10 11", nil) + + distances, err := getDistances(fakeSys, node) + assert.Nil(t, err) + assert.Len(t, distances, 2) + assert.Equal(t, uint64(10), distances[0]) + assert.Equal(t, uint64(11), distances[1]) +} + +func TestGetDistancesMissingDistances(t *testing.T) { + fakeSys := &fakesysfs.FakeSysFs{} + node := "/fakeSysfs/devices/system/node/node0" + fakeSys.SetDistances(node, "10 11", fmt.Errorf("no distances file")) + + distances, err := getDistances(fakeSys, node) + assert.Nil(t, err) + assert.Len(t, distances, 0) +}