From f14805695f7bfc5a11d72956a205c0e698834538 Mon Sep 17 00:00:00 2001 From: Matthew Wong Date: Thu, 7 Sep 2017 11:25:51 -0400 Subject: [PATCH 1/2] UPSTREAM: 51448: Add PVCRef to VolumeStats --- .../pkg/kubelet/apis/stats/v1alpha1/types.go | 9 ++ .../kubernetes/pkg/kubelet/server/stats/BUILD | 2 + .../server/stats/volume_stat_calculator.go | 27 +++- .../stats/volume_stat_calculator_test.go | 141 ++++++++++++++++++ 4 files changed, 174 insertions(+), 5 deletions(-) create mode 100644 vendor/k8s.io/kubernetes/pkg/kubelet/server/stats/volume_stat_calculator_test.go diff --git a/vendor/k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1/types.go b/vendor/k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1/types.go index e5cb0d9525e8..e08128658d2b 100644 --- a/vendor/k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1/types.go +++ b/vendor/k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1/types.go @@ -195,6 +195,15 @@ type VolumeStats struct { // Name is the name given to the Volume // +optional Name string `json:"name,omitempty"` + // Reference to the PVC, if one exists + // +optional + PVCRef *PVCReference `json:"pvcRef,omitempty"` +} + +// PVCReference contains enough information to describe the referenced PVC. +type PVCReference struct { + Name string `json:"name"` + Namespace string `json:"namespace"` } // FsStats contains data about filesystem usage. diff --git a/vendor/k8s.io/kubernetes/pkg/kubelet/server/stats/BUILD b/vendor/k8s.io/kubernetes/pkg/kubelet/server/stats/BUILD index 507718918418..5034330b5871 100644 --- a/vendor/k8s.io/kubernetes/pkg/kubelet/server/stats/BUILD +++ b/vendor/k8s.io/kubernetes/pkg/kubelet/server/stats/BUILD @@ -44,6 +44,7 @@ go_test( srcs = [ "mocks_test.go", "summary_test.go", + "volume_stat_calculator_test.go", ], library = ":go_default_library", tags = ["automanaged"], @@ -59,6 +60,7 @@ go_test( "//vendor/github.com/google/gofuzz:go_default_library", "//vendor/github.com/stretchr/testify/assert:go_default_library", "//vendor/github.com/stretchr/testify/mock:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", ], diff --git a/vendor/k8s.io/kubernetes/pkg/kubelet/server/stats/volume_stat_calculator.go b/vendor/k8s.io/kubernetes/pkg/kubelet/server/stats/volume_stat_calculator.go index d8d21513c813..018865a2af05 100644 --- a/vendor/k8s.io/kubernetes/pkg/kubelet/server/stats/volume_stat_calculator.go +++ b/vendor/k8s.io/kubernetes/pkg/kubelet/server/stats/volume_stat_calculator.go @@ -92,8 +92,14 @@ func (s *volumeStatCalculator) calcAndStoreStats() { return } + // Get volume specs for the pod - key'd by volume name + volumesSpec := make(map[string]v1.Volume) + for _, v := range s.pod.Spec.Volumes { + volumesSpec[v.Name] = v + } + // Call GetMetrics on each Volume and copy the result to a new VolumeStats.FsStats - stats := make([]stats.VolumeStats, 0, len(volumes)) + fsStats := make([]stats.VolumeStats, 0, len(volumes)) for name, v := range volumes { metric, err := v.GetMetrics() if err != nil { @@ -103,15 +109,25 @@ func (s *volumeStatCalculator) calcAndStoreStats() { } continue } - stats = append(stats, s.parsePodVolumeStats(name, metric)) + // Lookup the volume spec and add a 'PVCReference' for volumes that reference a PVC + volSpec := volumesSpec[name] + if pvcSource := volSpec.PersistentVolumeClaim; pvcSource != nil { + pvcRef := stats.PVCReference{ + Name: pvcSource.ClaimName, + Namespace: s.pod.GetNamespace(), + } + fsStats = append(fsStats, s.parsePodVolumeStats(name, &pvcRef, metric)) + } else { + fsStats = append(fsStats, s.parsePodVolumeStats(name, nil, metric)) + } } // Store the new stats - s.latest.Store(PodVolumeStats{Volumes: stats}) + s.latest.Store(PodVolumeStats{Volumes: fsStats}) } // parsePodVolumeStats converts (internal) volume.Metrics to (external) stats.VolumeStats structures -func (s *volumeStatCalculator) parsePodVolumeStats(podName string, metric *volume.Metrics) stats.VolumeStats { +func (s *volumeStatCalculator) parsePodVolumeStats(podName string, pvcRef *stats.PVCReference, metric *volume.Metrics) stats.VolumeStats { available := uint64(metric.Available.Value()) capacity := uint64(metric.Capacity.Value()) used := uint64(metric.Used.Value()) @@ -119,7 +135,8 @@ func (s *volumeStatCalculator) parsePodVolumeStats(podName string, metric *volum inodesFree := uint64(metric.InodesFree.Value()) inodesUsed := uint64(metric.InodesUsed.Value()) return stats.VolumeStats{ - Name: podName, + Name: podName, + PVCRef: pvcRef, FsStats: stats.FsStats{Time: metric.Time, AvailableBytes: &available, CapacityBytes: &capacity, UsedBytes: &used, Inodes: &inodes, InodesFree: &inodesFree, InodesUsed: &inodesUsed}, } diff --git a/vendor/k8s.io/kubernetes/pkg/kubelet/server/stats/volume_stat_calculator_test.go b/vendor/k8s.io/kubernetes/pkg/kubelet/server/stats/volume_stat_calculator_test.go new file mode 100644 index 000000000000..a6bd6f58fe5c --- /dev/null +++ b/vendor/k8s.io/kubernetes/pkg/kubelet/server/stats/volume_stat_calculator_test.go @@ -0,0 +1,141 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package stats + +import ( + "github.com/stretchr/testify/assert" + "testing" + "time" + + k8sv1 "k8s.io/kubernetes/pkg/api/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kubestats "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1" + "k8s.io/kubernetes/pkg/volume" +) + +const ( + namespace0 = "test0" + pName0 = "pod0" + capacity = int64(10000000) + available = int64(5000000) + inodesTotal = int64(2000) + inodesFree = int64(1000) + + vol0 = "vol0" + vol1 = "vol1" + pvcClaimName = "pvc-fake" +) + +func TestPVCRef(t *testing.T) { + // Create pod spec to test against + podVolumes := []k8sv1.Volume{ + { + Name: vol0, + VolumeSource: k8sv1.VolumeSource{ + GCEPersistentDisk: &k8sv1.GCEPersistentDiskVolumeSource{ + PDName: "fake-device1", + }, + }, + }, + { + Name: vol1, + VolumeSource: k8sv1.VolumeSource{ + PersistentVolumeClaim: &k8sv1.PersistentVolumeClaimVolumeSource{ + ClaimName: pvcClaimName, + }, + }, + }, + } + + fakePod := &k8sv1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: pName0, + Namespace: namespace0, + UID: "UID" + pName0, + }, + Spec: k8sv1.PodSpec{ + Volumes: podVolumes, + }, + } + + // Setup mock stats provider + mockStats := new(MockStatsProvider) + volumes := map[string]volume.Volume{vol0: &fakeVolume{}, vol1: &fakeVolume{}} + mockStats.On("ListVolumesForPod", fakePod.UID).Return(volumes, true) + + // Calculate stats for pod + statsCalculator := newVolumeStatCalculator(mockStats, time.Minute, fakePod) + statsCalculator.calcAndStoreStats() + vs, _ := statsCalculator.GetLatest() + + assert.Len(t, vs.Volumes, 2) + // Verify 'vol0' doesn't have a PVC reference + assert.Contains(t, vs.Volumes, kubestats.VolumeStats{ + Name: vol0, + FsStats: expectedFSStats(), + }) + // Verify 'vol1' has a PVC reference + assert.Contains(t, vs.Volumes, kubestats.VolumeStats{ + Name: vol1, + PVCRef: &kubestats.PVCReference{ + Name: pvcClaimName, + Namespace: namespace0, + }, + FsStats: expectedFSStats(), + }) +} + +// Fake volume/metrics provider +var _ volume.Volume = &fakeVolume{} + +type fakeVolume struct{} + +func (v *fakeVolume) GetPath() string { return "" } + +func (v *fakeVolume) GetMetrics() (*volume.Metrics, error) { + return expectedMetrics(), nil +} + +func expectedMetrics() *volume.Metrics { + return &volume.Metrics{ + Available: resource.NewQuantity(available, resource.BinarySI), + Capacity: resource.NewQuantity(capacity, resource.BinarySI), + Used: resource.NewQuantity(available-capacity, resource.BinarySI), + Inodes: resource.NewQuantity(inodesTotal, resource.BinarySI), + InodesFree: resource.NewQuantity(inodesFree, resource.BinarySI), + InodesUsed: resource.NewQuantity(inodesTotal-inodesFree, resource.BinarySI), + } +} + +func expectedFSStats() kubestats.FsStats { + metric := expectedMetrics() + available := uint64(metric.Available.Value()) + capacity := uint64(metric.Capacity.Value()) + used := uint64(metric.Used.Value()) + inodes := uint64(metric.Inodes.Value()) + inodesFree := uint64(metric.InodesFree.Value()) + inodesUsed := uint64(metric.InodesUsed.Value()) + return kubestats.FsStats{ + AvailableBytes: &available, + CapacityBytes: &capacity, + UsedBytes: &used, + Inodes: &inodes, + InodesFree: &inodesFree, + InodesUsed: &inodesUsed, + } +} From d33b06a9ad05fc4fbebed5ec9162cd418d0fbc71 Mon Sep 17 00:00:00 2001 From: Matthew Wong Date: Thu, 7 Sep 2017 11:28:45 -0400 Subject: [PATCH 2/2] UPSTREAM: 51553: Expose PVC metrics via kubelet prometheus --- .../kubernetes/pkg/kubelet/metrics/metrics.go | 84 ++++++++++++++++--- .../kubernetes/pkg/kubelet/server/stats/BUILD | 1 + .../server/stats/volume_stat_calculator.go | 14 ++++ 3 files changed, 87 insertions(+), 12 deletions(-) diff --git a/vendor/k8s.io/kubernetes/pkg/kubelet/metrics/metrics.go b/vendor/k8s.io/kubernetes/pkg/kubelet/metrics/metrics.go index 1b73082c0982..86b84c244fdf 100644 --- a/vendor/k8s.io/kubernetes/pkg/kubelet/metrics/metrics.go +++ b/vendor/k8s.io/kubernetes/pkg/kubelet/metrics/metrics.go @@ -26,18 +26,24 @@ import ( ) const ( - KubeletSubsystem = "kubelet" - PodWorkerLatencyKey = "pod_worker_latency_microseconds" - PodStartLatencyKey = "pod_start_latency_microseconds" - CgroupManagerOperationsKey = "cgroup_manager_latency_microseconds" - DockerOperationsLatencyKey = "docker_operations_latency_microseconds" - DockerOperationsKey = "docker_operations" - DockerOperationsErrorsKey = "docker_operations_errors" - DockerOperationsTimeoutKey = "docker_operations_timeout" - PodWorkerStartLatencyKey = "pod_worker_start_latency_microseconds" - PLEGRelistLatencyKey = "pleg_relist_latency_microseconds" - PLEGRelistIntervalKey = "pleg_relist_interval_microseconds" - EvictionStatsAgeKey = "eviction_stats_age_microseconds" + KubeletSubsystem = "kubelet" + PodWorkerLatencyKey = "pod_worker_latency_microseconds" + PodStartLatencyKey = "pod_start_latency_microseconds" + CgroupManagerOperationsKey = "cgroup_manager_latency_microseconds" + DockerOperationsLatencyKey = "docker_operations_latency_microseconds" + DockerOperationsKey = "docker_operations" + DockerOperationsErrorsKey = "docker_operations_errors" + DockerOperationsTimeoutKey = "docker_operations_timeout" + PodWorkerStartLatencyKey = "pod_worker_start_latency_microseconds" + PLEGRelistLatencyKey = "pleg_relist_latency_microseconds" + PLEGRelistIntervalKey = "pleg_relist_interval_microseconds" + EvictionStatsAgeKey = "eviction_stats_age_microseconds" + VolumeStatsCapacityBytesKey = "volume_stats_capacity_bytes" + VolumeStatsAvailableBytesKey = "volume_stats_available_bytes" + VolumeStatsUsedBytesKey = "volume_stats_used_bytes" + VolumeStatsInodesKey = "volume_stats_inodes" + VolumeStatsInodesFreeKey = "volume_stats_inodes_free" + VolumeStatsInodesUsedKey = "volume_stats_inodes_used" // Metrics keys of remote runtime operations RuntimeOperationsKey = "runtime_operations" RuntimeOperationsLatencyKey = "runtime_operations_latency_microseconds" @@ -162,6 +168,54 @@ var ( }, []string{"eviction_signal"}, ) + VolumeStatsCapacityBytes = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Subsystem: KubeletSubsystem, + Name: VolumeStatsCapacityBytesKey, + Help: "Capacity in bytes of the volume", + }, + []string{"namespace", "persistentvolumeclaim"}, + ) + VolumeStatsAvailableBytes = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Subsystem: KubeletSubsystem, + Name: VolumeStatsAvailableBytesKey, + Help: "Number of available bytes in the volume", + }, + []string{"namespace", "persistentvolumeclaim"}, + ) + VolumeStatsUsedBytes = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Subsystem: KubeletSubsystem, + Name: VolumeStatsUsedBytesKey, + Help: "Number of used bytes in the volume", + }, + []string{"namespace", "persistentvolumeclaim"}, + ) + VolumeStatsInodes = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Subsystem: KubeletSubsystem, + Name: VolumeStatsInodesKey, + Help: "Maximum number of inodes in the volume", + }, + []string{"namespace", "persistentvolumeclaim"}, + ) + VolumeStatsInodesFree = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Subsystem: KubeletSubsystem, + Name: VolumeStatsInodesFreeKey, + Help: "Number of free inodes in the volume", + }, + []string{"namespace", "persistentvolumeclaim"}, + ) + VolumeStatsInodesUsed = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Subsystem: KubeletSubsystem, + Name: VolumeStatsInodesUsedKey, + Help: "Number of used inodes in the volume", + }, + []string{"namespace", "persistentvolumeclaim"}, + ) ) var registerMetrics sync.Once @@ -186,6 +240,12 @@ func Register(containerCache kubecontainer.RuntimeCache) { prometheus.MustRegister(RuntimeOperationsLatency) prometheus.MustRegister(RuntimeOperationsErrors) prometheus.MustRegister(EvictionStatsAge) + prometheus.MustRegister(VolumeStatsCapacityBytes) + prometheus.MustRegister(VolumeStatsAvailableBytes) + prometheus.MustRegister(VolumeStatsUsedBytes) + prometheus.MustRegister(VolumeStatsInodes) + prometheus.MustRegister(VolumeStatsInodesFree) + prometheus.MustRegister(VolumeStatsInodesUsed) }) } diff --git a/vendor/k8s.io/kubernetes/pkg/kubelet/server/stats/BUILD b/vendor/k8s.io/kubernetes/pkg/kubelet/server/stats/BUILD index 5034330b5871..111d6f8e7e0c 100644 --- a/vendor/k8s.io/kubernetes/pkg/kubelet/server/stats/BUILD +++ b/vendor/k8s.io/kubernetes/pkg/kubelet/server/stats/BUILD @@ -25,6 +25,7 @@ go_library( "//pkg/kubelet/cm:go_default_library", "//pkg/kubelet/container:go_default_library", "//pkg/kubelet/leaky:go_default_library", + "//pkg/kubelet/metrics:go_default_library", "//pkg/kubelet/network:go_default_library", "//pkg/kubelet/types:go_default_library", "//pkg/kubelet/util/format:go_default_library", diff --git a/vendor/k8s.io/kubernetes/pkg/kubelet/server/stats/volume_stat_calculator.go b/vendor/k8s.io/kubernetes/pkg/kubelet/server/stats/volume_stat_calculator.go index 018865a2af05..f3a7299ea613 100644 --- a/vendor/k8s.io/kubernetes/pkg/kubelet/server/stats/volume_stat_calculator.go +++ b/vendor/k8s.io/kubernetes/pkg/kubelet/server/stats/volume_stat_calculator.go @@ -24,6 +24,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "k8s.io/kubernetes/pkg/api/v1" stats "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1" + "k8s.io/kubernetes/pkg/kubelet/metrics" "k8s.io/kubernetes/pkg/kubelet/util/format" "k8s.io/kubernetes/pkg/volume" @@ -85,6 +86,7 @@ func (s *volumeStatCalculator) GetLatest() (PodVolumeStats, bool) { } // calcAndStoreStats calculates PodVolumeStats for a given pod and writes the result to the s.latest cache. +// If the pod references PVCs, the prometheus metrics for those are updated with the result. func (s *volumeStatCalculator) calcAndStoreStats() { // Find all Volumes for the Pod volumes, found := s.statsProvider.ListVolumesForPod(s.pod.UID) @@ -117,6 +119,8 @@ func (s *volumeStatCalculator) calcAndStoreStats() { Namespace: s.pod.GetNamespace(), } fsStats = append(fsStats, s.parsePodVolumeStats(name, &pvcRef, metric)) + // Set the PVC's prometheus metrics + s.setPVCMetrics(&pvcRef, metric) } else { fsStats = append(fsStats, s.parsePodVolumeStats(name, nil, metric)) } @@ -141,3 +145,13 @@ func (s *volumeStatCalculator) parsePodVolumeStats(podName string, pvcRef *stats UsedBytes: &used, Inodes: &inodes, InodesFree: &inodesFree, InodesUsed: &inodesUsed}, } } + +// setPVCMetrics sets the given PVC's prometheus metrics to match the given volume.Metrics +func (s *volumeStatCalculator) setPVCMetrics(pvcRef *stats.PVCReference, metric *volume.Metrics) { + metrics.VolumeStatsAvailableBytes.WithLabelValues(pvcRef.Namespace, pvcRef.Name).Set(float64(metric.Available.Value())) + metrics.VolumeStatsCapacityBytes.WithLabelValues(pvcRef.Namespace, pvcRef.Name).Set(float64(metric.Capacity.Value())) + metrics.VolumeStatsUsedBytes.WithLabelValues(pvcRef.Namespace, pvcRef.Name).Set(float64(metric.Used.Value())) + metrics.VolumeStatsInodes.WithLabelValues(pvcRef.Namespace, pvcRef.Name).Set(float64(metric.Inodes.Value())) + metrics.VolumeStatsInodesFree.WithLabelValues(pvcRef.Namespace, pvcRef.Name).Set(float64(metric.InodesFree.Value())) + metrics.VolumeStatsInodesUsed.WithLabelValues(pvcRef.Namespace, pvcRef.Name).Set(float64(metric.InodesUsed.Value())) +}