diff --git a/cmd/controller/controller.go b/cmd/controller/controller.go index 2ed0dc96405..faf30e00f7d 100644 --- a/cmd/controller/controller.go +++ b/cmd/controller/controller.go @@ -3,12 +3,12 @@ package controller import ( "context" "fmt" + "net" "net/http" "net/http/pprof" "os" "time" - "github.com/prometheus/client_golang/prometheus/promhttp" v1 "k8s.io/api/authorization/v1" apiv1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -18,36 +18,25 @@ import ( "k8s.io/client-go/tools/leaderelection/resourcelock" "k8s.io/client-go/tools/record" "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/manager/signals" kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" "github.com/kubeovn/kube-ovn/pkg/controller" - "github.com/kubeovn/kube-ovn/pkg/server" + "github.com/kubeovn/kube-ovn/pkg/metrics" "github.com/kubeovn/kube-ovn/pkg/util" "github.com/kubeovn/kube-ovn/versions" ) -const ( - svcName = "kube-ovn-controller" - ovnLeaderResource = "kube-ovn-controller" -) +const ovnLeaderResource = "kube-ovn-controller" func CmdMain() { defer klog.Flush() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - go func() { - stopCh := signals.SetupSignalHandler().Done() - <-stopCh - cancel() - }() + ctx := signals.SetupSignalHandler() klog.Infof(versions.String()) - controller.InitClientGoMetrics() - controller.InitWorkQueueMetrics() - util.InitKlogMetrics() config, err := controller.ParseFlags() if err != nil { util.LogFatalAndExit(err, "failed to parse config") @@ -59,36 +48,47 @@ func CmdMain() { utilruntime.Must(kubeovnv1.AddToScheme(scheme.Scheme)) go func() { - mux := http.NewServeMux() - if config.EnableMetrics { - mux.Handle("/metrics", promhttp.Handler()) - } if config.EnablePprof { + mux := http.NewServeMux() mux.HandleFunc("/debug/pprof/", pprof.Index) mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) mux.HandleFunc("/debug/pprof/profile", pprof.Profile) mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) mux.HandleFunc("/debug/pprof/trace", pprof.Trace) - } - addr := util.JoinHostPort(util.GetDefaultListenAddr(), config.PprofPort) - if !config.SecureServing { - server := &http.Server{ - Addr: addr, - ReadHeaderTimeout: 3 * time.Second, - Handler: mux, - } - util.LogFatalAndExit(server.ListenAndServe(), "failed to listen and server on %s", server.Addr) - } else { - ch, err := server.SecureServing(addr, svcName, mux) + listerner, err := net.ListenTCP("tcp", &net.TCPAddr{IP: net.ParseIP("127.0.0.1"), Port: int(config.PprofPort)}) if err != nil { - util.LogFatalAndExit(err, "failed to serve on %s", addr) + util.LogFatalAndExit(err, "failed to listen on %s", util.JoinHostPort("127.0.0.1", config.PprofPort)) } - <-ch + svr := manager.Server{ + Name: "pprof", + Server: &http.Server{ + Handler: mux, + MaxHeaderBytes: 1 << 20, + IdleTimeout: 90 * time.Second, + ReadHeaderTimeout: 32 * time.Second, + }, + Listener: listerner, + } + go func() { + if err = svr.Start(ctx); err != nil { + util.LogFatalAndExit(err, "failed to run pprof server") + } + }() + } + + if !config.EnableMetrics { + return + } + metrics.InitKlogMetrics() + metrics.InitClientGoMetrics() + addr := util.JoinHostPort(util.GetDefaultListenAddr(), config.PprofPort) + if err := metrics.Run(ctx, config.KubeRestConfig, addr, config.SecureServing); err != nil { + util.LogFatalAndExit(err, "failed to run metrics server") } + <-ctx.Done() }() - // ctx, cancel := context.WithCancel(context.Background()) recorder := record.NewBroadcaster().NewRecorder(scheme.Scheme, apiv1.EventSource{ Component: ovnLeaderResource, Host: os.Getenv(util.HostnameEnv), diff --git a/cmd/daemon/cniserver.go b/cmd/daemon/cniserver.go index 8d06b9818a3..1138a781d57 100644 --- a/cmd/daemon/cniserver.go +++ b/cmd/daemon/cniserver.go @@ -2,6 +2,7 @@ package daemon import ( "fmt" + "net" "net/http" "net/http/pprof" "os" @@ -9,27 +10,25 @@ import ( "strings" "time" - "github.com/prometheus/client_golang/prometheus/promhttp" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" kubeinformers "k8s.io/client-go/informers" "k8s.io/klog/v2" - "k8s.io/sample-controller/pkg/signals" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/manager/signals" kubeovninformer "github.com/kubeovn/kube-ovn/pkg/client/informers/externalversions" "github.com/kubeovn/kube-ovn/pkg/daemon" + "github.com/kubeovn/kube-ovn/pkg/metrics" "github.com/kubeovn/kube-ovn/pkg/ovs" - "github.com/kubeovn/kube-ovn/pkg/server" "github.com/kubeovn/kube-ovn/pkg/util" "github.com/kubeovn/kube-ovn/versions" ) -const svcName = "kube-ovn-cni" - func CmdMain() { defer klog.Flush() daemon.InitMetrics() - util.InitKlogMetrics() + metrics.InitKlogMetrics() config := daemon.ParseFlags() klog.Infof(versions.String()) @@ -62,7 +61,8 @@ func CmdMain() { util.LogFatalAndExit(err, "failed to do the OS initialization") } - stopCh := signals.SetupSignalHandler().Done() + ctx := signals.SetupSignalHandler() + stopCh := ctx.Done() podInformerFactory := kubeinformers.NewSharedInformerFactoryWithOptions(config.KubeClient, 0, kubeinformers.WithTweakListOptions(func(listOption *v1.ListOptions) { listOption.FieldSelector = fmt.Sprintf("spec.nodeName=%s", config.NodeName) @@ -87,18 +87,6 @@ func CmdMain() { util.LogFatalAndExit(err, "failed to mv cni config file") } - mux := http.NewServeMux() - if config.EnableMetrics { - mux.Handle("/metrics", promhttp.Handler()) - } - if config.EnablePprof { - mux.HandleFunc("/debug/pprof/", pprof.Index) - mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) - mux.HandleFunc("/debug/pprof/profile", pprof.Profile) - mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) - mux.HandleFunc("/debug/pprof/trace", pprof.Trace) - } - addr := util.GetDefaultListenAddr() if config.EnableVerboseConnCheck { go func() { @@ -116,21 +104,40 @@ func CmdMain() { }() } - listenAddr := util.JoinHostPort(addr, config.PprofPort) - if !config.SecureServing { - server := &http.Server{ - Addr: listenAddr, - ReadHeaderTimeout: 3 * time.Second, - Handler: mux, - } - util.LogFatalAndExit(server.ListenAndServe(), "failed to listen and server on %s", server.Addr) - } else { - ch, err := server.SecureServing(listenAddr, svcName, mux) + if config.EnablePprof { + mux := http.NewServeMux() + mux.HandleFunc("/debug/pprof/", pprof.Index) + mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) + mux.HandleFunc("/debug/pprof/profile", pprof.Profile) + mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) + mux.HandleFunc("/debug/pprof/trace", pprof.Trace) + + listerner, err := net.ListenTCP("tcp", &net.TCPAddr{IP: net.ParseIP("127.0.0.1"), Port: int(config.PprofPort)}) if err != nil { - util.LogFatalAndExit(err, "failed to serve on %s", listenAddr) + util.LogFatalAndExit(err, "failed to listen on %s", util.JoinHostPort("127.0.0.1", config.PprofPort)) + } + svr := manager.Server{ + Name: "pprof", + Server: &http.Server{ + Handler: mux, + MaxHeaderBytes: 1 << 20, + IdleTimeout: 90 * time.Second, + ReadHeaderTimeout: 32 * time.Second, + }, + Listener: listerner, } - <-ch + go func() { + if err = svr.Start(ctx); err != nil { + util.LogFatalAndExit(err, "failed to run pprof server") + } + }() + } + + listenAddr := util.JoinHostPort(addr, config.PprofPort) + if err = metrics.Run(ctx, nil, listenAddr, config.SecureServing); err != nil { + util.LogFatalAndExit(err, "failed to run metrics server") } + <-stopCh } func mvCNIConf(configDir, configFile, confName string) error { diff --git a/cmd/ovn_monitor/ovn_monitor.go b/cmd/ovn_monitor/ovn_monitor.go index d9979a5d998..648d4572ab1 100644 --- a/cmd/ovn_monitor/ovn_monitor.go +++ b/cmd/ovn_monitor/ovn_monitor.go @@ -1,22 +1,18 @@ package ovn_monitor import ( - "net/http" "os" "strings" - "time" - "github.com/prometheus/client_golang/prometheus/promhttp" "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/manager/signals" + "github.com/kubeovn/kube-ovn/pkg/metrics" ovn "github.com/kubeovn/kube-ovn/pkg/ovnmonitor" - "github.com/kubeovn/kube-ovn/pkg/server" "github.com/kubeovn/kube-ovn/pkg/util" "github.com/kubeovn/kube-ovn/versions" ) -const svcName = "kube-ovn-monitor" - const port = 10661 func CmdMain() { @@ -41,24 +37,10 @@ func CmdMain() { go exporter.TryClientConnection() } exporter.StartOvnMetrics() - mux := http.NewServeMux() - if config.EnableMetrics { - mux.Handle(config.MetricsPath, promhttp.Handler()) - klog.Infoln("Listening on", addr) - } - if !config.SecureServing { - server := &http.Server{ - Addr: addr, - ReadHeaderTimeout: 3 * time.Second, - Handler: mux, - } - util.LogFatalAndExit(server.ListenAndServe(), "failed to listen and server on %s", addr) - } else { - ch, err := server.SecureServing(addr, svcName, mux) - if err != nil { - util.LogFatalAndExit(err, "failed to serve on %s", addr) - } - <-ch + ctx := signals.SetupSignalHandler() + if err = metrics.Run(ctx, nil, addr, config.SecureServing); err != nil { + util.LogFatalAndExit(err, "failed to run metrics server") } + <-ctx.Done() } diff --git a/cmd/pinger/pinger.go b/cmd/pinger/pinger.go index a3f7d5c6772..b7d68cd1616 100644 --- a/cmd/pinger/pinger.go +++ b/cmd/pinger/pinger.go @@ -1,13 +1,12 @@ package pinger import ( - "net/http" _ "net/http/pprof" // #nosec - "time" - "github.com/prometheus/client_golang/prometheus/promhttp" "k8s.io/klog/v2" + "k8s.io/sample-controller/pkg/signals" + "github.com/kubeovn/kube-ovn/pkg/metrics" "github.com/kubeovn/kube-ovn/pkg/pinger" "github.com/kubeovn/kube-ovn/pkg/util" "github.com/kubeovn/kube-ovn/versions" @@ -23,20 +22,14 @@ func CmdMain() { } if config.Mode == "server" { if config.EnableMetrics { - pinger.InitPingerMetrics() - util.InitKlogMetrics() - - mux := http.NewServeMux() - mux.Handle("/metrics", promhttp.Handler()) go func() { - // conform to Gosec G114 - // https://github.com/securego/gosec#available-rules - server := &http.Server{ - Addr: util.JoinHostPort("0.0.0.0", config.Port), - ReadHeaderTimeout: 3 * time.Second, - Handler: mux, + pinger.InitPingerMetrics() + metrics.InitKlogMetrics() + ctx := signals.SetupSignalHandler() + if err := metrics.Run(ctx, nil, util.JoinHostPort("0.0.0.0", config.Port), false); err != nil { + util.LogFatalAndExit(err, "failed to run metrics server") } - util.LogFatalAndExit(server.ListenAndServe(), "failed to listen and serve on %s", server.Addr) + <-ctx.Done() }() } diff --git a/go.mod b/go.mod index 0138f1d10b5..302073e29a6 100644 --- a/go.mod +++ b/go.mod @@ -44,7 +44,6 @@ require ( gopkg.in/k8snetworkplumbingwg/multus-cni.v4 v4.0.2 k8s.io/api v0.30.3 k8s.io/apimachinery v0.30.3 - k8s.io/apiserver v0.30.3 k8s.io/client-go v12.0.0+incompatible k8s.io/klog/v2 v2.130.1 k8s.io/kubectl v0.30.3 @@ -243,6 +242,7 @@ require ( gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.30.3 // indirect + k8s.io/apiserver v0.30.3 // indirect k8s.io/cli-runtime v0.30.3 // indirect k8s.io/cloud-provider v0.30.3 // indirect k8s.io/cluster-bootstrap v0.30.3 // indirect diff --git a/pkg/controller/client_go_adapter.go b/pkg/controller/client_go_adapter.go deleted file mode 100644 index 7211db793a8..00000000000 --- a/pkg/controller/client_go_adapter.go +++ /dev/null @@ -1,210 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// This file comes from sigs.k8s.io/controller-runtime/pkg/metrics/client_go_adapter.go - -package controller - -import ( - "context" - "net/url" - "strings" - "time" - - "github.com/prometheus/client_golang/prometheus" - reflectormetrics "k8s.io/client-go/tools/cache" - clientmetrics "k8s.io/client-go/tools/metrics" -) - -// this file contains setup logic to initialize the myriad of places -// that client-go registers metrics. we copy the names and formats -// from Kubernetes so that we match the core controllers. - -var ( - // client metrics - - requestLatency = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Name: "rest_client_request_latency_seconds", - Help: "Request latency in seconds. Broken down by verb and URL.", - Buckets: prometheus.ExponentialBuckets(0.001, 2, 10), - }, - []string{"verb", "url"}, - ) - - requestResult = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "rest_client_requests_total", - Help: "Number of HTTP requests, partitioned by status code, method, and host.", - }, - []string{"code", "method", "host"}, - ) - - // reflector metrics - - // TODO(directxman12): update these to be histograms once the metrics overhaul KEP - // PRs start landing. - - reflectorSubsystem = "reflector" - - listsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ - Subsystem: reflectorSubsystem, - Name: "lists_total", - Help: "Total number of API lists done by the reflectors", - }, []string{"name"}) - - listsDuration = prometheus.NewSummaryVec(prometheus.SummaryOpts{ - Subsystem: reflectorSubsystem, - Name: "list_duration_seconds", - Help: "How long an API list takes to return and decode for the reflectors", - }, []string{"name"}) - - itemsPerList = prometheus.NewSummaryVec(prometheus.SummaryOpts{ - Subsystem: reflectorSubsystem, - Name: "items_per_list", - Help: "How many items an API list returns to the reflectors", - }, []string{"name"}) - - watchesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ - Subsystem: reflectorSubsystem, - Name: "watches_total", - Help: "Total number of API watches done by the reflectors", - }, []string{"name"}) - - shortWatchesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ - Subsystem: reflectorSubsystem, - Name: "short_watches_total", - Help: "Total number of short API watches done by the reflectors", - }, []string{"name"}) - - watchDuration = prometheus.NewSummaryVec(prometheus.SummaryOpts{ - Subsystem: reflectorSubsystem, - Name: "watch_duration_seconds", - Help: "How long an API watch takes to return and decode for the reflectors", - }, []string{"name"}) - - itemsPerWatch = prometheus.NewSummaryVec(prometheus.SummaryOpts{ - Subsystem: reflectorSubsystem, - Name: "items_per_watch", - Help: "How many items an API watch returns to the reflectors", - }, []string{"name"}) - - lastResourceVersion = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Subsystem: reflectorSubsystem, - Name: "last_resource_version", - Help: "Last resource version seen for the reflectors", - }, []string{"name"}) -) - -func InitClientGoMetrics() { - registerClientMetrics() - registerReflectorMetrics() -} - -// registerClientMetrics sets up the client latency metrics from client-go -func registerClientMetrics() { - // register the metrics with our registry - prometheus.MustRegister(requestLatency) - prometheus.MustRegister(requestResult) - - // register the metrics with client-go - opts := clientmetrics.RegisterOpts{ - RequestLatency: clientmetrics.LatencyMetric(&latencyAdapter{metric: requestLatency}), - RequestResult: clientmetrics.ResultMetric(&resultAdapter{metric: requestResult}), - } - clientmetrics.Register(opts) -} - -// registerReflectorMetrics sets up reflector (reconcile) loop metrics -func registerReflectorMetrics() { - prometheus.MustRegister(listsTotal) - prometheus.MustRegister(listsDuration) - prometheus.MustRegister(itemsPerList) - prometheus.MustRegister(watchesTotal) - prometheus.MustRegister(shortWatchesTotal) - prometheus.MustRegister(watchDuration) - prometheus.MustRegister(itemsPerWatch) - prometheus.MustRegister(lastResourceVersion) - - reflectormetrics.SetReflectorMetricsProvider(reflectorMetricsProvider{}) -} - -// this section contains adapters, implementations, and other sundry organic, artisanally -// hand-crafted syntax trees required to convince client-go that it actually wants to let -// someone use its metrics. - -// Client metrics adapters (method #1 for client-go metrics), -// copied (more-or-less directly) from k8s.io/kubernetes setup code -// (which isn't anywhere in an easily-importable place). - -type latencyAdapter struct { - metric *prometheus.HistogramVec -} - -func (l *latencyAdapter) Observe(_ context.Context, verb string, u url.URL, latency time.Duration) { - url := u.String() - last := strings.LastIndex(url, "/") - if last != -1 { - url = url[:last] - } - l.metric.WithLabelValues(verb, url).Observe(latency.Seconds()) -} - -type resultAdapter struct { - metric *prometheus.CounterVec -} - -func (r *resultAdapter) Increment(_ context.Context, code, method, host string) { - r.metric.WithLabelValues(code, method, host).Inc() -} - -// Reflector metrics provider (method #2 for client-go metrics), -// copied (more-or-less directly) from k8s.io/kubernetes setup code -// (which isn't anywhere in an easily-importable place). - -type reflectorMetricsProvider struct{} - -func (reflectorMetricsProvider) NewListsMetric(name string) reflectormetrics.CounterMetric { - return listsTotal.WithLabelValues(name) -} - -func (reflectorMetricsProvider) NewListDurationMetric(name string) reflectormetrics.SummaryMetric { - return listsDuration.WithLabelValues(name) -} - -func (reflectorMetricsProvider) NewItemsInListMetric(name string) reflectormetrics.SummaryMetric { - return itemsPerList.WithLabelValues(name) -} - -func (reflectorMetricsProvider) NewWatchesMetric(name string) reflectormetrics.CounterMetric { - return watchesTotal.WithLabelValues(name) -} - -func (reflectorMetricsProvider) NewShortWatchesMetric(name string) reflectormetrics.CounterMetric { - return shortWatchesTotal.WithLabelValues(name) -} - -func (reflectorMetricsProvider) NewWatchDurationMetric(name string) reflectormetrics.SummaryMetric { - return watchDuration.WithLabelValues(name) -} - -func (reflectorMetricsProvider) NewItemsInWatchMetric(name string) reflectormetrics.SummaryMetric { - return itemsPerWatch.WithLabelValues(name) -} - -func (reflectorMetricsProvider) NewLastResourceVersionMetric(name string) reflectormetrics.GaugeMetric { - return lastResourceVersion.WithLabelValues(name) -} diff --git a/pkg/controller/net_metrics.go b/pkg/controller/net_metrics.go index f0c24238909..fb9f7479211 100644 --- a/pkg/controller/net_metrics.go +++ b/pkg/controller/net_metrics.go @@ -1,6 +1,9 @@ package controller -import "github.com/prometheus/client_golang/prometheus" +import ( + "github.com/prometheus/client_golang/prometheus" + "sigs.k8s.io/controller-runtime/pkg/metrics" +) var ( metricSubnetAvailableIPs = prometheus.NewGaugeVec( @@ -66,9 +69,9 @@ var ( ) func registerMetrics() { - prometheus.MustRegister(metricSubnetAvailableIPs) - prometheus.MustRegister(metricSubnetUsedIPs) - prometheus.MustRegister(metricCentralSubnetInfo) - prometheus.MustRegister(metricSubnetIPAMInfo) - prometheus.MustRegister(metricSubnetIPAssignedInfo) + metrics.Registry.MustRegister(metricSubnetAvailableIPs) + metrics.Registry.MustRegister(metricSubnetUsedIPs) + metrics.Registry.MustRegister(metricCentralSubnetInfo) + metrics.Registry.MustRegister(metricSubnetIPAMInfo) + metrics.Registry.MustRegister(metricSubnetIPAssignedInfo) } diff --git a/pkg/controller/workqueue_metrics.go b/pkg/controller/workqueue_metrics.go deleted file mode 100644 index eebb6217074..00000000000 --- a/pkg/controller/workqueue_metrics.go +++ /dev/null @@ -1,170 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// This file comes from sigs.k8s.io/controller-runtime/pkg/metrics/workqueue.go - -package controller - -import ( - "github.com/prometheus/client_golang/prometheus" - "k8s.io/client-go/util/workqueue" -) - -// This file is copied and adapted from k8s.io/kubernetes/pkg/util/workqueue/prometheus -// which registers metrics to the default prometheus Registry. We require very -// similar functionality, but must register metrics to a different Registry. - -func InitWorkQueueMetrics() { - workqueue.SetProvider(workqueueMetricsProvider{}) -} - -func registerWorkqueueMetric(c prometheus.Collector, _, _ string) { - prometheus.MustRegister(c) -} - -type workqueueMetricsProvider struct{} - -func (workqueueMetricsProvider) NewDepthMetric(queue string) workqueue.GaugeMetric { - const name = "workqueue_depth" - m := prometheus.NewGauge(prometheus.GaugeOpts{ - Name: name, - Help: "Current depth of workqueue", - ConstLabels: prometheus.Labels{"name": queue}, - }) - registerWorkqueueMetric(m, name, queue) - return m -} - -func (workqueueMetricsProvider) NewAddsMetric(queue string) workqueue.CounterMetric { - const name = "workqueue_adds_total" - m := prometheus.NewCounter(prometheus.CounterOpts{ - Name: name, - Help: "Total number of adds handled by workqueue", - ConstLabels: prometheus.Labels{"name": queue}, - }) - registerWorkqueueMetric(m, name, queue) - return m -} - -func (workqueueMetricsProvider) NewLatencyMetric(queue string) workqueue.HistogramMetric { - const name = "workqueue_queue_duration_seconds" - m := prometheus.NewHistogram(prometheus.HistogramOpts{ - Name: name, - Help: "How long in seconds an item stays in workqueue before being requested.", - ConstLabels: prometheus.Labels{"name": queue}, - Buckets: prometheus.ExponentialBuckets(10e-9, 10, 10), - }) - registerWorkqueueMetric(m, name, queue) - return m -} - -func (workqueueMetricsProvider) NewWorkDurationMetric(queue string) workqueue.HistogramMetric { - const name = "workqueue_work_duration_seconds" - m := prometheus.NewHistogram(prometheus.HistogramOpts{ - Name: name, - Help: "How long in seconds processing an item from workqueue takes.", - ConstLabels: prometheus.Labels{"name": queue}, - Buckets: prometheus.ExponentialBuckets(10e-9, 10, 10), - }) - registerWorkqueueMetric(m, name, queue) - return m -} - -func (workqueueMetricsProvider) NewUnfinishedWorkSecondsMetric(queue string) workqueue.SettableGaugeMetric { - const name = "workqueue_unfinished_work_seconds" - m := prometheus.NewGauge(prometheus.GaugeOpts{ - Name: name, - Help: "How many seconds of work has done that " + - "is in progress and hasn't been observed by work_duration. Large " + - "values indicate stuck threads. One can deduce the number of stuck " + - "threads by observing the rate at which this increases.", - ConstLabels: prometheus.Labels{"name": queue}, - }) - registerWorkqueueMetric(m, name, queue) - return m -} - -func (workqueueMetricsProvider) NewLongestRunningProcessorSecondsMetric(queue string) workqueue.SettableGaugeMetric { - const name = "workqueue_longest_running_processor_seconds" - m := prometheus.NewGauge(prometheus.GaugeOpts{ - Name: name, - Help: "How many seconds has the longest running " + - "processor for workqueue been running.", - ConstLabels: prometheus.Labels{"name": queue}, - }) - registerWorkqueueMetric(m, name, queue) - return m -} - -func (workqueueMetricsProvider) NewRetriesMetric(queue string) workqueue.CounterMetric { - const name = "workqueue_retries_total" - m := prometheus.NewCounter(prometheus.CounterOpts{ - Name: name, - Help: "Total number of retries handled by workqueue", - ConstLabels: prometheus.Labels{"name": queue}, - }) - registerWorkqueueMetric(m, name, queue) - return m -} - -// TODO(abursavich): Remove the following deprecated metrics when they are -// removed from k8s.io/client-go/util/workqueue. - -func (workqueueMetricsProvider) NewDeprecatedLongestRunningProcessorMicrosecondsMetric(queue string) workqueue.SettableGaugeMetric { - const name = "workqueue_longest_running_processor_microseconds" - m := prometheus.NewGauge(prometheus.GaugeOpts{ - Name: name, - Help: "(Deprecated) How many microseconds has the longest running " + - "processor for workqueue been running.", - ConstLabels: prometheus.Labels{"name": queue}, - }) - registerWorkqueueMetric(m, name, queue) - return m -} - -// NOTE: The following deprecated metrics are noops because they were never -// included in controller-runtime. - -func (workqueueMetricsProvider) NewDeprecatedDepthMetric(_ string) workqueue.GaugeMetric { - return noopMetric{} -} - -func (workqueueMetricsProvider) NewDeprecatedAddsMetric(_ string) workqueue.CounterMetric { - return noopMetric{} -} - -func (workqueueMetricsProvider) NewDeprecatedLatencyMetric(_ string) workqueue.SummaryMetric { - return noopMetric{} -} - -func (workqueueMetricsProvider) NewDeprecatedWorkDurationMetric(_ string) workqueue.SummaryMetric { - return noopMetric{} -} - -func (workqueueMetricsProvider) NewDeprecatedUnfinishedWorkSecondsMetric(_ string) workqueue.SettableGaugeMetric { - return noopMetric{} -} - -func (workqueueMetricsProvider) NewDeprecatedRetriesMetric(_ string) workqueue.CounterMetric { - return noopMetric{} -} - -type noopMetric struct{} - -func (noopMetric) Inc() {} -func (noopMetric) Dec() {} -func (noopMetric) Set(float64) {} -func (noopMetric) Observe(float64) {} diff --git a/pkg/daemon/metrics.go b/pkg/daemon/metrics.go index b6b4162b79b..45cb8bd37be 100644 --- a/pkg/daemon/metrics.go +++ b/pkg/daemon/metrics.go @@ -1,14 +1,8 @@ package daemon import ( - "context" - "net/url" - "strings" - "time" - "github.com/prometheus/client_golang/prometheus" - reflectormetrics "k8s.io/client-go/tools/cache" - clientmetrics "k8s.io/client-go/tools/metrics" + "sigs.k8s.io/controller-runtime/pkg/metrics" ) var ( @@ -48,24 +42,6 @@ var ( []string{"node_name"}, ) - // client metrics - requestLatency = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Name: "rest_client_request_latency_seconds", - Help: "Request latency in seconds. Broken down by verb and URL.", - Buckets: prometheus.ExponentialBuckets(0.001, 2, 10), - }, - []string{"verb", "url"}, - ) - - requestResult = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "rest_client_requests_total", - Help: "Number of HTTP requests, partitioned by status code, method, and host.", - }, - []string{"code", "method", "host"}, - ) - metricOvnSubnetGatewayPacketBytes = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "ovn_subnet_gateway_packet_bytes", @@ -169,182 +145,30 @@ var ( }, []string{ "hostname", }) - - // reflector metrics - - // TODO(directxman12): update these to be histograms once the metrics overhaul KEP - // PRs start landing. - - reflectorSubsystem = "reflector" - - listsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ - Subsystem: reflectorSubsystem, - Name: "lists_total", - Help: "Total number of API lists done by the reflectors", - }, []string{"name"}) - - listsDuration = prometheus.NewSummaryVec(prometheus.SummaryOpts{ - Subsystem: reflectorSubsystem, - Name: "list_duration_seconds", - Help: "How long an API list takes to return and decode for the reflectors", - }, []string{"name"}) - - itemsPerList = prometheus.NewSummaryVec(prometheus.SummaryOpts{ - Subsystem: reflectorSubsystem, - Name: "items_per_list", - Help: "How many items an API list returns to the reflectors", - }, []string{"name"}) - - watchesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ - Subsystem: reflectorSubsystem, - Name: "watches_total", - Help: "Total number of API watches done by the reflectors", - }, []string{"name"}) - - shortWatchesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ - Subsystem: reflectorSubsystem, - Name: "short_watches_total", - Help: "Total number of short API watches done by the reflectors", - }, []string{"name"}) - - watchDuration = prometheus.NewSummaryVec(prometheus.SummaryOpts{ - Subsystem: reflectorSubsystem, - Name: "watch_duration_seconds", - Help: "How long an API watch takes to return and decode for the reflectors", - }, []string{"name"}) - - itemsPerWatch = prometheus.NewSummaryVec(prometheus.SummaryOpts{ - Subsystem: reflectorSubsystem, - Name: "items_per_watch", - Help: "How many items an API watch returns to the reflectors", - }, []string{"name"}) - - lastResourceVersion = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Subsystem: reflectorSubsystem, - Name: "last_resource_version", - Help: "Last resource version seen for the reflectors", - }, []string{"name"}) ) func InitMetrics() { - registerReflectorMetrics() - registerClientMetrics() registerOvnSubnetGatewayMetrics() registerSystemParameterMetrics() - prometheus.MustRegister(cniOperationHistogram) - prometheus.MustRegister(cniWaitAddressResult) - prometheus.MustRegister(cniConnectivityResult) + metrics.Registry.MustRegister(cniOperationHistogram) + metrics.Registry.MustRegister(cniWaitAddressResult) + metrics.Registry.MustRegister(cniConnectivityResult) } func registerOvnSubnetGatewayMetrics() { - prometheus.MustRegister(metricOvnSubnetGatewayPacketBytes) - prometheus.MustRegister(metricOvnSubnetGatewayPackets) + metrics.Registry.MustRegister(metricOvnSubnetGatewayPacketBytes) + metrics.Registry.MustRegister(metricOvnSubnetGatewayPackets) } func registerSystemParameterMetrics() { - prometheus.MustRegister(metricIPLocalPortRange) - prometheus.MustRegister(metricCheckSumErr) - prometheus.MustRegister(metricCniConfig) - prometheus.MustRegister(metricDNSSearch) - prometheus.MustRegister(metricTCPTwRecycle) - prometheus.MustRegister(metricTCPMtuProbing) - prometheus.MustRegister(metricConntrackTCPLiberal) - prometheus.MustRegister(metricBridgeNfCallIptables) - prometheus.MustRegister(metricTCPMem) - prometheus.MustRegister(metricIPv6RouteMaxsize) -} - -// registerClientMetrics sets up the client latency metrics from client-go -func registerClientMetrics() { - // register the metrics with our registry - prometheus.MustRegister(requestLatency) - prometheus.MustRegister(requestResult) - - // register the metrics with client-go - opts := clientmetrics.RegisterOpts{ - RequestLatency: clientmetrics.LatencyMetric(&latencyAdapter{metric: requestLatency}), - RequestResult: clientmetrics.ResultMetric(&resultAdapter{metric: requestResult}), - } - clientmetrics.Register(opts) -} - -// registerReflectorMetrics sets up reflector (reconcile) loop metrics -func registerReflectorMetrics() { - prometheus.MustRegister(listsTotal) - prometheus.MustRegister(listsDuration) - prometheus.MustRegister(itemsPerList) - prometheus.MustRegister(watchesTotal) - prometheus.MustRegister(shortWatchesTotal) - prometheus.MustRegister(watchDuration) - prometheus.MustRegister(itemsPerWatch) - prometheus.MustRegister(lastResourceVersion) - - reflectormetrics.SetReflectorMetricsProvider(reflectorMetricsProvider{}) -} - -// this section contains adapters, implementations, and other sundry organic, artisanally -// hand-crafted syntax trees required to convince client-go that it actually wants to let -// someone use its metrics. - -// Client metrics adapters (method #1 for client-go metrics), -// copied (more-or-less directly) from k8s.io/kubernetes setup code -// (which isn't anywhere in an easily-importable place). - -type latencyAdapter struct { - metric *prometheus.HistogramVec -} - -func (l *latencyAdapter) Observe(_ context.Context, verb string, u url.URL, latency time.Duration) { - url := u.String() - last := strings.LastIndex(url, "/") - if last != -1 { - url = url[:last] - } - l.metric.WithLabelValues(verb, url).Observe(latency.Seconds()) -} - -type resultAdapter struct { - metric *prometheus.CounterVec -} - -func (r *resultAdapter) Increment(_ context.Context, code, method, host string) { - r.metric.WithLabelValues(code, method, host).Inc() -} - -// Reflector metrics provider (method #2 for client-go metrics), -// copied (more-or-less directly) from k8s.io/kubernetes setup code -// (which isn't anywhere in an easily-importable place). - -type reflectorMetricsProvider struct{} - -func (reflectorMetricsProvider) NewListsMetric(name string) reflectormetrics.CounterMetric { - return listsTotal.WithLabelValues(name) -} - -func (reflectorMetricsProvider) NewListDurationMetric(name string) reflectormetrics.SummaryMetric { - return listsDuration.WithLabelValues(name) -} - -func (reflectorMetricsProvider) NewItemsInListMetric(name string) reflectormetrics.SummaryMetric { - return itemsPerList.WithLabelValues(name) -} - -func (reflectorMetricsProvider) NewWatchesMetric(name string) reflectormetrics.CounterMetric { - return watchesTotal.WithLabelValues(name) -} - -func (reflectorMetricsProvider) NewShortWatchesMetric(name string) reflectormetrics.CounterMetric { - return shortWatchesTotal.WithLabelValues(name) -} - -func (reflectorMetricsProvider) NewWatchDurationMetric(name string) reflectormetrics.SummaryMetric { - return watchDuration.WithLabelValues(name) -} - -func (reflectorMetricsProvider) NewItemsInWatchMetric(name string) reflectormetrics.SummaryMetric { - return itemsPerWatch.WithLabelValues(name) -} - -func (reflectorMetricsProvider) NewLastResourceVersionMetric(name string) reflectormetrics.GaugeMetric { - return lastResourceVersion.WithLabelValues(name) + metrics.Registry.MustRegister(metricIPLocalPortRange) + metrics.Registry.MustRegister(metricCheckSumErr) + metrics.Registry.MustRegister(metricCniConfig) + metrics.Registry.MustRegister(metricDNSSearch) + metrics.Registry.MustRegister(metricTCPTwRecycle) + metrics.Registry.MustRegister(metricTCPMtuProbing) + metrics.Registry.MustRegister(metricConntrackTCPLiberal) + metrics.Registry.MustRegister(metricBridgeNfCallIptables) + metrics.Registry.MustRegister(metricTCPMem) + metrics.Registry.MustRegister(metricIPv6RouteMaxsize) } diff --git a/pkg/metrics/client_go_adapter.go b/pkg/metrics/client_go_adapter.go new file mode 100644 index 00000000000..08829879cbb --- /dev/null +++ b/pkg/metrics/client_go_adapter.go @@ -0,0 +1,60 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// This file comes from sigs.k8s.io/controller-runtime/pkg/metrics/client_go_adapter.go + +package metrics + +import ( + "context" + "net/url" + "path" + "time" + + "github.com/prometheus/client_golang/prometheus" + "k8s.io/client-go/tools/metrics" + ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" +) + +var requestLatency = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "rest_client_request_latency_seconds", + Help: "Request latency in seconds. Broken down by verb and URL.", + Buckets: prometheus.ExponentialBuckets(0.001, 2, 10), + }, + []string{"verb", "url"}, +) + +func InitClientGoMetrics() { + registerClientMetrics() +} + +// registerClientMetrics sets up the client latency metrics from client-go +func registerClientMetrics() { + // register the metrics with our registry + ctrlmetrics.Registry.MustRegister(requestLatency) + + // register the metrics with client-go + metrics.RequestLatency = &latencyAdapter{metric: requestLatency} +} + +type latencyAdapter struct { + metric *prometheus.HistogramVec +} + +func (l *latencyAdapter) Observe(_ context.Context, verb string, u url.URL, latency time.Duration) { + l.metric.WithLabelValues(verb, path.Dir(u.Path)).Observe(latency.Seconds()) +} diff --git a/pkg/util/klog_metrics.go b/pkg/metrics/klog.go similarity index 87% rename from pkg/util/klog_metrics.go rename to pkg/metrics/klog.go index e4c31763699..1226ecfb2e0 100644 --- a/pkg/util/klog_metrics.go +++ b/pkg/metrics/klog.go @@ -1,8 +1,10 @@ -package util +package metrics import ( "time" + "sigs.k8s.io/controller-runtime/pkg/metrics" + "github.com/prometheus/client_golang/prometheus" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/klog/v2" @@ -25,8 +27,8 @@ func InitKlogMetrics() { } func registerKlogMetrics() { - prometheus.MustRegister(klogLinesGaugeVec) - prometheus.MustRegister(klogBytesGaugeVec) + metrics.Registry.MustRegister(klogLinesGaugeVec) + metrics.Registry.MustRegister(klogBytesGaugeVec) } func fetchKlogMetrics() { diff --git a/pkg/metrics/server.go b/pkg/metrics/server.go new file mode 100644 index 00000000000..5a16762df69 --- /dev/null +++ b/pkg/metrics/server.go @@ -0,0 +1,38 @@ +package metrics + +import ( + "context" + "fmt" + + "k8s.io/client-go/rest" + "k8s.io/klog/v2" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/metrics/filters" + "sigs.k8s.io/controller-runtime/pkg/metrics/server" +) + +func Run(ctx context.Context, config *rest.Config, addr string, secureServing bool) error { + if config == nil { + config = ctrl.GetConfigOrDie() + } + client, err := rest.HTTPClientFor(config) + if err != nil { + klog.Error(err) + return fmt.Errorf("failed to create http client: %v", err) + } + + options := server.Options{ + SecureServing: secureServing, + BindAddress: addr, + } + if secureServing { + options.FilterProvider = filters.WithAuthenticationAndAuthorization + } + svr, err := server.NewServer(options, config, client) + if err != nil { + klog.Error(err) + return fmt.Errorf("failed to create metrics server: %v", err) + } + + return svr.Start(ctx) +} diff --git a/pkg/ovnmonitor/metric.go b/pkg/ovnmonitor/metric.go index 30f7634dc68..834125dc579 100644 --- a/pkg/ovnmonitor/metric.go +++ b/pkg/ovnmonitor/metric.go @@ -1,6 +1,9 @@ package ovnmonitor -import "github.com/prometheus/client_golang/prometheus" +import ( + "github.com/prometheus/client_golang/prometheus" + "sigs.k8s.io/controller-runtime/pkg/metrics" +) var ( // OVN basic info @@ -476,48 +479,48 @@ var ( func registerOvnMetrics() { // ovn status metrics - prometheus.MustRegister(metricOvnHealthyStatus) - prometheus.MustRegister(metricOvnHealthyStatusContent) - prometheus.MustRegister(metricRequestErrorNums) - prometheus.MustRegister(metricLogFileSize) - prometheus.MustRegister(metricDBFileSize) - prometheus.MustRegister(metricDBStatus) + metrics.Registry.MustRegister(metricOvnHealthyStatus) + metrics.Registry.MustRegister(metricOvnHealthyStatusContent) + metrics.Registry.MustRegister(metricRequestErrorNums) + metrics.Registry.MustRegister(metricLogFileSize) + metrics.Registry.MustRegister(metricDBFileSize) + metrics.Registry.MustRegister(metricDBStatus) // ovn chassis metrics - prometheus.MustRegister(metricChassisInfo) - prometheus.MustRegister(metricLogicalSwitchInfo) - prometheus.MustRegister(metricLogicalSwitchExternalIDs) - prometheus.MustRegister(metricLogicalSwitchPortBinding) - prometheus.MustRegister(metricLogicalSwitchTunnelKey) - prometheus.MustRegister(metricLogicalSwitchPortsNum) - prometheus.MustRegister(metricLogicalSwitchPortInfo) - prometheus.MustRegister(metricLogicalSwitchPortTunnelKey) + metrics.Registry.MustRegister(metricChassisInfo) + metrics.Registry.MustRegister(metricLogicalSwitchInfo) + metrics.Registry.MustRegister(metricLogicalSwitchExternalIDs) + metrics.Registry.MustRegister(metricLogicalSwitchPortBinding) + metrics.Registry.MustRegister(metricLogicalSwitchTunnelKey) + metrics.Registry.MustRegister(metricLogicalSwitchPortsNum) + metrics.Registry.MustRegister(metricLogicalSwitchPortInfo) + metrics.Registry.MustRegister(metricLogicalSwitchPortTunnelKey) // OVN Cluster basic info metrics - prometheus.MustRegister(metricClusterEnabled) - prometheus.MustRegister(metricClusterRole) - prometheus.MustRegister(metricClusterStatus) - prometheus.MustRegister(metricClusterTerm) - - prometheus.MustRegister(metricClusterLeaderSelf) - prometheus.MustRegister(metricClusterVoteSelf) - prometheus.MustRegister(metricClusterElectionTimer) - prometheus.MustRegister(metricClusterNotCommittedEntryCount) - prometheus.MustRegister(metricClusterNotAppliedEntryCount) - - prometheus.MustRegister(metricClusterLogIndexStart) - prometheus.MustRegister(metricClusterLogIndexNext) - prometheus.MustRegister(metricClusterInConnTotal) - prometheus.MustRegister(metricClusterOutConnTotal) - prometheus.MustRegister(metricClusterInConnErrTotal) - prometheus.MustRegister(metricClusterOutConnErrTotal) + metrics.Registry.MustRegister(metricClusterEnabled) + metrics.Registry.MustRegister(metricClusterRole) + metrics.Registry.MustRegister(metricClusterStatus) + metrics.Registry.MustRegister(metricClusterTerm) + + metrics.Registry.MustRegister(metricClusterLeaderSelf) + metrics.Registry.MustRegister(metricClusterVoteSelf) + metrics.Registry.MustRegister(metricClusterElectionTimer) + metrics.Registry.MustRegister(metricClusterNotCommittedEntryCount) + metrics.Registry.MustRegister(metricClusterNotAppliedEntryCount) + + metrics.Registry.MustRegister(metricClusterLogIndexStart) + metrics.Registry.MustRegister(metricClusterLogIndexNext) + metrics.Registry.MustRegister(metricClusterInConnTotal) + metrics.Registry.MustRegister(metricClusterOutConnTotal) + metrics.Registry.MustRegister(metricClusterInConnErrTotal) + metrics.Registry.MustRegister(metricClusterOutConnErrTotal) // to be implemented - prometheus.MustRegister(metricClusterPeerNextIndex) - prometheus.MustRegister(metricClusterPeerMatchIndex) - prometheus.MustRegister(metricClusterNextIndex) - prometheus.MustRegister(metricClusterMatchIndex) - prometheus.MustRegister(metricClusterPeerInConnInfo) - prometheus.MustRegister(metricClusterPeerOutConnInfo) - prometheus.MustRegister(metricClusterPeerCount) + metrics.Registry.MustRegister(metricClusterPeerNextIndex) + metrics.Registry.MustRegister(metricClusterPeerMatchIndex) + metrics.Registry.MustRegister(metricClusterNextIndex) + metrics.Registry.MustRegister(metricClusterMatchIndex) + metrics.Registry.MustRegister(metricClusterPeerInConnInfo) + metrics.Registry.MustRegister(metricClusterPeerOutConnInfo) + metrics.Registry.MustRegister(metricClusterPeerCount) } diff --git a/pkg/ovs/adapter.go b/pkg/ovs/adapter.go index a2315b68194..63c33e57e57 100644 --- a/pkg/ovs/adapter.go +++ b/pkg/ovs/adapter.go @@ -1,6 +1,9 @@ package ovs -import "github.com/prometheus/client_golang/prometheus" +import ( + "github.com/prometheus/client_golang/prometheus" + "sigs.k8s.io/controller-runtime/pkg/metrics" +) // OVN NB metrics var ovsClientRequestLatency = prometheus.NewHistogramVec( @@ -16,5 +19,5 @@ func init() { } func registerOvsClientMetrics() { - prometheus.MustRegister(ovsClientRequestLatency) + metrics.Registry.MustRegister(ovsClientRequestLatency) } diff --git a/pkg/pinger/metrics.go b/pkg/pinger/metrics.go index eef0fe1be6f..da36519a389 100644 --- a/pkg/pinger/metrics.go +++ b/pkg/pinger/metrics.go @@ -1,6 +1,9 @@ package pinger -import "github.com/prometheus/client_golang/prometheus" +import ( + "github.com/prometheus/client_golang/prometheus" + "sigs.k8s.io/controller-runtime/pkg/metrics" +) var ( ovsUpGauge = prometheus.NewGaugeVec( @@ -639,73 +642,73 @@ var ( ) func InitPingerMetrics() { - prometheus.MustRegister(ovsUpGauge) - prometheus.MustRegister(ovsDownGauge) - prometheus.MustRegister(ovnControllerUpGauge) - prometheus.MustRegister(ovnControllerDownGauge) - prometheus.MustRegister(inconsistentPortBindingGauge) - prometheus.MustRegister(apiserverHealthyGauge) - prometheus.MustRegister(apiserverUnhealthyGauge) - prometheus.MustRegister(apiserverRequestLatencyHistogram) - prometheus.MustRegister(internalDNSHealthyGauge) - prometheus.MustRegister(internalDNSUnhealthyGauge) - prometheus.MustRegister(internalDNSRequestLatencyHistogram) - prometheus.MustRegister(externalDNSHealthyGauge) - prometheus.MustRegister(externalDNSUnhealthyGauge) - prometheus.MustRegister(externalDNSRequestLatencyHistogram) - prometheus.MustRegister(podPingLatencyHistogram) - prometheus.MustRegister(podPingLostCounter) - prometheus.MustRegister(podPingTotalCounter) - prometheus.MustRegister(nodePingLatencyHistogram) - prometheus.MustRegister(nodePingLostCounter) - prometheus.MustRegister(nodePingTotalCounter) - prometheus.MustRegister(externalPingLatencyHistogram) - prometheus.MustRegister(externalPingLostCounter) + metrics.Registry.MustRegister(ovsUpGauge) + metrics.Registry.MustRegister(ovsDownGauge) + metrics.Registry.MustRegister(ovnControllerUpGauge) + metrics.Registry.MustRegister(ovnControllerDownGauge) + metrics.Registry.MustRegister(inconsistentPortBindingGauge) + metrics.Registry.MustRegister(apiserverHealthyGauge) + metrics.Registry.MustRegister(apiserverUnhealthyGauge) + metrics.Registry.MustRegister(apiserverRequestLatencyHistogram) + metrics.Registry.MustRegister(internalDNSHealthyGauge) + metrics.Registry.MustRegister(internalDNSUnhealthyGauge) + metrics.Registry.MustRegister(internalDNSRequestLatencyHistogram) + metrics.Registry.MustRegister(externalDNSHealthyGauge) + metrics.Registry.MustRegister(externalDNSUnhealthyGauge) + metrics.Registry.MustRegister(externalDNSRequestLatencyHistogram) + metrics.Registry.MustRegister(podPingLatencyHistogram) + metrics.Registry.MustRegister(podPingLostCounter) + metrics.Registry.MustRegister(podPingTotalCounter) + metrics.Registry.MustRegister(nodePingLatencyHistogram) + metrics.Registry.MustRegister(nodePingLostCounter) + metrics.Registry.MustRegister(nodePingTotalCounter) + metrics.Registry.MustRegister(externalPingLatencyHistogram) + metrics.Registry.MustRegister(externalPingLostCounter) // ovs status metrics - prometheus.MustRegister(metricOvsHealthyStatus) - prometheus.MustRegister(metricOvsInfo) - prometheus.MustRegister(metricRequestErrorNums) - prometheus.MustRegister(metricLogFileSize) - prometheus.MustRegister(metricDbFileSize) + metrics.Registry.MustRegister(metricOvsHealthyStatus) + metrics.Registry.MustRegister(metricOvsInfo) + metrics.Registry.MustRegister(metricRequestErrorNums) + metrics.Registry.MustRegister(metricLogFileSize) + metrics.Registry.MustRegister(metricDbFileSize) // ovs datapath metrics - prometheus.MustRegister(metricOvsDp) - prometheus.MustRegister(metricOvsDpTotal) - prometheus.MustRegister(metricOvsDpIf) - prometheus.MustRegister(metricOvsDpIfTotal) - prometheus.MustRegister(metricOvsDpFlowsTotal) - prometheus.MustRegister(metricOvsDpFlowsLookupHit) - prometheus.MustRegister(metricOvsDpFlowsLookupMissed) - prometheus.MustRegister(metricOvsDpFlowsLookupLost) - prometheus.MustRegister(metricOvsDpMasksHit) - prometheus.MustRegister(metricOvsDpMasksTotal) - prometheus.MustRegister(metricOvsDpMasksHitRatio) + metrics.Registry.MustRegister(metricOvsDp) + metrics.Registry.MustRegister(metricOvsDpTotal) + metrics.Registry.MustRegister(metricOvsDpIf) + metrics.Registry.MustRegister(metricOvsDpIfTotal) + metrics.Registry.MustRegister(metricOvsDpFlowsTotal) + metrics.Registry.MustRegister(metricOvsDpFlowsLookupHit) + metrics.Registry.MustRegister(metricOvsDpFlowsLookupMissed) + metrics.Registry.MustRegister(metricOvsDpFlowsLookupLost) + metrics.Registry.MustRegister(metricOvsDpMasksHit) + metrics.Registry.MustRegister(metricOvsDpMasksTotal) + metrics.Registry.MustRegister(metricOvsDpMasksHitRatio) // ovs Interface basic info metrics - prometheus.MustRegister(interfaceMain) - prometheus.MustRegister(interfaceAdminState) - prometheus.MustRegister(interfaceLinkState) - prometheus.MustRegister(interfaceMacInUse) - prometheus.MustRegister(interfaceMtu) - prometheus.MustRegister(interfaceOfPort) - prometheus.MustRegister(interfaceIfIndex) + metrics.Registry.MustRegister(interfaceMain) + metrics.Registry.MustRegister(interfaceAdminState) + metrics.Registry.MustRegister(interfaceLinkState) + metrics.Registry.MustRegister(interfaceMacInUse) + metrics.Registry.MustRegister(interfaceMtu) + metrics.Registry.MustRegister(interfaceOfPort) + metrics.Registry.MustRegister(interfaceIfIndex) // ovs Interface statistics metrics - prometheus.MustRegister(interfaceStatTxPackets) - prometheus.MustRegister(interfaceStatTxBytes) - prometheus.MustRegister(interfaceStatRxPackets) - prometheus.MustRegister(interfaceStatRxBytes) - prometheus.MustRegister(interfaceStatRxCrcError) - prometheus.MustRegister(interfaceStatRxDropped) - prometheus.MustRegister(interfaceStatRxErrorsTotal) - prometheus.MustRegister(interfaceStatRxFrameError) - prometheus.MustRegister(interfaceStatRxMissedError) - prometheus.MustRegister(interfaceStatRxOverrunError) - prometheus.MustRegister(interfaceStatTxDropped) - prometheus.MustRegister(interfaceStatTxErrorsTotal) - prometheus.MustRegister(interfaceStatCollisions) - prometheus.MustRegister(interfaceStatRxMulticastPackets) + metrics.Registry.MustRegister(interfaceStatTxPackets) + metrics.Registry.MustRegister(interfaceStatTxBytes) + metrics.Registry.MustRegister(interfaceStatRxPackets) + metrics.Registry.MustRegister(interfaceStatRxBytes) + metrics.Registry.MustRegister(interfaceStatRxCrcError) + metrics.Registry.MustRegister(interfaceStatRxDropped) + metrics.Registry.MustRegister(interfaceStatRxErrorsTotal) + metrics.Registry.MustRegister(interfaceStatRxFrameError) + metrics.Registry.MustRegister(interfaceStatRxMissedError) + metrics.Registry.MustRegister(interfaceStatRxOverrunError) + metrics.Registry.MustRegister(interfaceStatTxDropped) + metrics.Registry.MustRegister(interfaceStatTxErrorsTotal) + metrics.Registry.MustRegister(interfaceStatCollisions) + metrics.Registry.MustRegister(interfaceStatRxMulticastPackets) } func SetOvsUpMetrics(nodeName string) { diff --git a/pkg/server/server.go b/pkg/server/server.go deleted file mode 100644 index 90f511df42c..00000000000 --- a/pkg/server/server.go +++ /dev/null @@ -1,101 +0,0 @@ -package server - -import ( - "fmt" - "net" - "net/http" - "os" - "strconv" - "strings" - - "k8s.io/apiserver/pkg/endpoints/filters" - "k8s.io/apiserver/pkg/endpoints/request" - "k8s.io/apiserver/pkg/server" - "k8s.io/apiserver/pkg/server/options" - "k8s.io/client-go/rest" - "k8s.io/klog/v2" - - "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned/scheme" -) - -func SecureServing(addr, svcName string, handler http.Handler) (<-chan struct{}, error) { - host, port, err := net.SplitHostPort(addr) - if err != nil { - klog.Error(err) - return nil, fmt.Errorf("invalid listen address %q: %v", addr, err) - } - - namespace := os.Getenv("POD_NAMESPACE") - podName := os.Getenv("POD_NAME") - podIPs := os.Getenv("POD_IPS") - alternateDNS := []string{podName, svcName, fmt.Sprintf("%s.%s", svcName, namespace), fmt.Sprintf("%s.%s.svc", svcName, namespace)} - alternateIPs := []net.IP{net.ParseIP("127.0.0.1"), net.IPv6loopback} - for _, podIP := range strings.Split(podIPs, ",") { - if ip := net.ParseIP(podIP); ip != nil { - alternateIPs = append(alternateIPs, ip) - } - } - - var clientConfig *rest.Config - opt := options.NewSecureServingOptions().WithLoopback() - authnOpt := options.NewDelegatingAuthenticationOptions() - authzOpt := options.NewDelegatingAuthorizationOptions() - opt.ServerCert.PairName = svcName - opt.ServerCert.CertDirectory = "" - authnOpt.RemoteKubeConfigFileOptional = true - authzOpt.RemoteKubeConfigFileOptional = true - - if host != "" { - ip := net.ParseIP(host) - if ip == nil { - err = fmt.Errorf("invalid listen address: %q", addr) - klog.Error(err) - return nil, err - } - opt.BindAddress = ip - p, err := strconv.Atoi(port) - if err != nil { - klog.Error(err) - return nil, fmt.Errorf("invalid listen address %q: %v", addr, err) - } - opt.BindPort = p - } - - if err = opt.MaybeDefaultWithSelfSignedCerts("localhost", alternateDNS, alternateIPs); err != nil { - klog.Error(err) - return nil, fmt.Errorf("failed to genarate self signed certificates: %v", err) - } - - var serving *server.SecureServingInfo - var authn server.AuthenticationInfo - var authz server.AuthorizationInfo - if err = opt.ApplyTo(&serving, &clientConfig); err != nil { - klog.Error(err) - return nil, fmt.Errorf("failed to apply secure serving options to secure serving info: %v", err) - } - if err = authnOpt.ApplyTo(&authn, serving, nil); err != nil { - klog.Error(err) - return nil, fmt.Errorf("failed to apply authn options to authn info: %v", err) - } - if err = authzOpt.ApplyTo(&authz); err != nil { - klog.Error(err) - return nil, fmt.Errorf("failed to apply authz options to authz info: %v", err) - } - - handler = filters.WithAuthorization(handler, authz.Authorizer, scheme.Codecs) - handler = filters.WithAuthentication(handler, authn.Authenticator, filters.Unauthorized(scheme.Codecs), nil, nil) - - requestInfoResolver := &request.RequestInfoFactory{} - handler = filters.WithRequestInfo(handler, requestInfoResolver) - handler = filters.WithCacheControl(handler) - server.AuthorizeClientBearerToken(clientConfig, &authn, &authz) - - stopCh := make(chan struct{}, 1) - _, listenerStoppedCh, err := serving.Serve(handler, 0, stopCh) - if err != nil { - klog.Error(err) - return nil, fmt.Errorf("failed to serve on %s: %v", addr, err) - } - - return listenerStoppedCh, nil -}