refactor metrics (#4313)

Signed-off-by: zhangzujian <[email protected]>
kubeovn · Jul 22, 2024 · d00cf67 · d00cf67
1 parent de6525e
commit d00cf67
Show file tree

Hide file tree

Showing 16 changed files with 325 additions and 888 deletions.
diff --git a/cmd/controller/controller.go b/cmd/controller/controller.go
@@ -3,12 +3,12 @@ package controller
 import (
 	"context"
 	"fmt"
+	"net"
 	"net/http"
 	"net/http/pprof"
 	"os"
 	"time"
 
-	"github.com/prometheus/client_golang/prometheus/promhttp"
 	v1 "k8s.io/api/authorization/v1"
 	apiv1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -18,36 +18,25 @@ import (
 	"k8s.io/client-go/tools/leaderelection/resourcelock"
 	"k8s.io/client-go/tools/record"
 	"k8s.io/klog/v2"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
 	"sigs.k8s.io/controller-runtime/pkg/manager/signals"
 
 	kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
 	"github.com/kubeovn/kube-ovn/pkg/controller"
-	"github.com/kubeovn/kube-ovn/pkg/server"
+	"github.com/kubeovn/kube-ovn/pkg/metrics"
 	"github.com/kubeovn/kube-ovn/pkg/util"
 	"github.com/kubeovn/kube-ovn/versions"
 )
 
-const (
-	svcName           = "kube-ovn-controller"
-	ovnLeaderResource = "kube-ovn-controller"
-)
+const ovnLeaderResource = "kube-ovn-controller"
 
 func CmdMain() {
 	defer klog.Flush()
 
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-	go func() {
-		stopCh := signals.SetupSignalHandler().Done()
-		<-stopCh
-		cancel()
-	}()
+	ctx := signals.SetupSignalHandler()
 
 	klog.Infof(versions.String())
 
-	controller.InitClientGoMetrics()
-	controller.InitWorkQueueMetrics()
-	util.InitKlogMetrics()
 	config, err := controller.ParseFlags()
 	if err != nil {
 		util.LogFatalAndExit(err, "failed to parse config")
@@ -59,36 +48,47 @@ func CmdMain() {
 	utilruntime.Must(kubeovnv1.AddToScheme(scheme.Scheme))
 
 	go func() {
-		mux := http.NewServeMux()
-		if config.EnableMetrics {
-			mux.Handle("/metrics", promhttp.Handler())
-		}
 		if config.EnablePprof {
+			mux := http.NewServeMux()
 			mux.HandleFunc("/debug/pprof/", pprof.Index)
 			mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
 			mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
 			mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
 			mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
-		}
 
-		addr := util.JoinHostPort(util.GetDefaultListenAddr(), config.PprofPort)
-		if !config.SecureServing {
-			server := &http.Server{
-				Addr:              addr,
-				ReadHeaderTimeout: 3 * time.Second,
-				Handler:           mux,
-			}
-			util.LogFatalAndExit(server.ListenAndServe(), "failed to listen and server on %s", server.Addr)
-		} else {
-			ch, err := server.SecureServing(addr, svcName, mux)
+			listerner, err := net.ListenTCP("tcp", &net.TCPAddr{IP: net.ParseIP("127.0.0.1"), Port: int(config.PprofPort)})
 			if err != nil {
-				util.LogFatalAndExit(err, "failed to serve on %s", addr)
+				util.LogFatalAndExit(err, "failed to listen on %s", util.JoinHostPort("127.0.0.1", config.PprofPort))
 			}
-			<-ch
+			svr := manager.Server{
+				Name: "pprof",
+				Server: &http.Server{
+					Handler:           mux,
+					MaxHeaderBytes:    1 << 20,
+					IdleTimeout:       90 * time.Second,
+					ReadHeaderTimeout: 32 * time.Second,
+				},
+				Listener: listerner,
+			}
+			go func() {
+				if err = svr.Start(ctx); err != nil {
+					util.LogFatalAndExit(err, "failed to run pprof server")
+				}
+			}()
+		}
+
+		if !config.EnableMetrics {
+			return
+		}
+		metrics.InitKlogMetrics()
+		metrics.InitClientGoMetrics()
+		addr := util.JoinHostPort(util.GetDefaultListenAddr(), config.PprofPort)
+		if err := metrics.Run(ctx, config.KubeRestConfig, addr, config.SecureServing); err != nil {
+			util.LogFatalAndExit(err, "failed to run metrics server")
 		}
+		<-ctx.Done()
 	}()
 
-	//	ctx, cancel := context.WithCancel(context.Background())
 	recorder := record.NewBroadcaster().NewRecorder(scheme.Scheme, apiv1.EventSource{
 		Component: ovnLeaderResource,
 		Host:      os.Getenv(util.HostnameEnv),

diff --git a/cmd/daemon/cniserver.go b/cmd/daemon/cniserver.go
@@ -2,34 +2,33 @@ package daemon
 
 import (
 	"fmt"
+	"net"
 	"net/http"
 	"net/http/pprof"
 	"os"
 	"path/filepath"
 	"strings"
 	"time"
 
-	"github.com/prometheus/client_golang/prometheus/promhttp"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	kubeinformers "k8s.io/client-go/informers"
 	"k8s.io/klog/v2"
-	"k8s.io/sample-controller/pkg/signals"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+	"sigs.k8s.io/controller-runtime/pkg/manager/signals"
 
 	kubeovninformer "github.com/kubeovn/kube-ovn/pkg/client/informers/externalversions"
 	"github.com/kubeovn/kube-ovn/pkg/daemon"
+	"github.com/kubeovn/kube-ovn/pkg/metrics"
 	"github.com/kubeovn/kube-ovn/pkg/ovs"
-	"github.com/kubeovn/kube-ovn/pkg/server"
 	"github.com/kubeovn/kube-ovn/pkg/util"
 	"github.com/kubeovn/kube-ovn/versions"
 )
 
-const svcName = "kube-ovn-cni"
-
 func CmdMain() {
 	defer klog.Flush()
 
 	daemon.InitMetrics()
-	util.InitKlogMetrics()
+	metrics.InitKlogMetrics()
 
 	config := daemon.ParseFlags()
 	klog.Infof(versions.String())
@@ -62,7 +61,8 @@ func CmdMain() {
 		util.LogFatalAndExit(err, "failed to do the OS initialization")
 	}
 
-	stopCh := signals.SetupSignalHandler().Done()
+	ctx := signals.SetupSignalHandler()
+	stopCh := ctx.Done()
 	podInformerFactory := kubeinformers.NewSharedInformerFactoryWithOptions(config.KubeClient, 0,
 		kubeinformers.WithTweakListOptions(func(listOption *v1.ListOptions) {
 			listOption.FieldSelector = fmt.Sprintf("spec.nodeName=%s", config.NodeName)
@@ -87,18 +87,6 @@ func CmdMain() {
 		util.LogFatalAndExit(err, "failed to mv cni config file")
 	}
 
-	mux := http.NewServeMux()
-	if config.EnableMetrics {
-		mux.Handle("/metrics", promhttp.Handler())
-	}
-	if config.EnablePprof {
-		mux.HandleFunc("/debug/pprof/", pprof.Index)
-		mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
-		mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
-		mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
-		mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
-	}
-
 	addr := util.GetDefaultListenAddr()
 	if config.EnableVerboseConnCheck {
 		go func() {
@@ -116,21 +104,40 @@ func CmdMain() {
 		}()
 	}
 
-	listenAddr := util.JoinHostPort(addr, config.PprofPort)
-	if !config.SecureServing {
-		server := &http.Server{
-			Addr:              listenAddr,
-			ReadHeaderTimeout: 3 * time.Second,
-			Handler:           mux,
-		}
-		util.LogFatalAndExit(server.ListenAndServe(), "failed to listen and server on %s", server.Addr)
-	} else {
-		ch, err := server.SecureServing(listenAddr, svcName, mux)
+	if config.EnablePprof {
+		mux := http.NewServeMux()
+		mux.HandleFunc("/debug/pprof/", pprof.Index)
+		mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
+		mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
+		mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
+		mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
+
+		listerner, err := net.ListenTCP("tcp", &net.TCPAddr{IP: net.ParseIP("127.0.0.1"), Port: int(config.PprofPort)})
 		if err != nil {
-			util.LogFatalAndExit(err, "failed to serve on %s", listenAddr)
+			util.LogFatalAndExit(err, "failed to listen on %s", util.JoinHostPort("127.0.0.1", config.PprofPort))
+		}
+		svr := manager.Server{
+			Name: "pprof",
+			Server: &http.Server{
+				Handler:           mux,
+				MaxHeaderBytes:    1 << 20,
+				IdleTimeout:       90 * time.Second,
+				ReadHeaderTimeout: 32 * time.Second,
+			},
+			Listener: listerner,
 		}
-		<-ch
+		go func() {
+			if err = svr.Start(ctx); err != nil {
+				util.LogFatalAndExit(err, "failed to run pprof server")
+			}
+		}()
+	}
+
+	listenAddr := util.JoinHostPort(addr, config.PprofPort)
+	if err = metrics.Run(ctx, nil, listenAddr, config.SecureServing); err != nil {
+		util.LogFatalAndExit(err, "failed to run metrics server")
 	}
+	<-stopCh
 }
 
 func mvCNIConf(configDir, configFile, confName string) error {

diff --git a/cmd/ovn_monitor/ovn_monitor.go b/cmd/ovn_monitor/ovn_monitor.go
@@ -1,22 +1,18 @@
 package ovn_monitor
 
 import (
-	"net/http"
 	"os"
 	"strings"
-	"time"
 
-	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"k8s.io/klog/v2"
+	"sigs.k8s.io/controller-runtime/pkg/manager/signals"
 
+	"github.com/kubeovn/kube-ovn/pkg/metrics"
 	ovn "github.com/kubeovn/kube-ovn/pkg/ovnmonitor"
-	"github.com/kubeovn/kube-ovn/pkg/server"
 	"github.com/kubeovn/kube-ovn/pkg/util"
 	"github.com/kubeovn/kube-ovn/versions"
 )
 
-const svcName = "kube-ovn-monitor"
-
 const port = 10661
 
 func CmdMain() {
@@ -41,24 +37,10 @@ func CmdMain() {
 		go exporter.TryClientConnection()
 	}
 	exporter.StartOvnMetrics()
-	mux := http.NewServeMux()
-	if config.EnableMetrics {
-		mux.Handle(config.MetricsPath, promhttp.Handler())
-		klog.Infoln("Listening on", addr)
-	}
 
-	if !config.SecureServing {
-		server := &http.Server{
-			Addr:              addr,
-			ReadHeaderTimeout: 3 * time.Second,
-			Handler:           mux,
-		}
-		util.LogFatalAndExit(server.ListenAndServe(), "failed to listen and server on %s", addr)
-	} else {
-		ch, err := server.SecureServing(addr, svcName, mux)
-		if err != nil {
-			util.LogFatalAndExit(err, "failed to serve on %s", addr)
-		}
-		<-ch
+	ctx := signals.SetupSignalHandler()
+	if err = metrics.Run(ctx, nil, addr, config.SecureServing); err != nil {
+		util.LogFatalAndExit(err, "failed to run metrics server")
 	}
+	<-ctx.Done()
 }
diff --git a/cmd/pinger/pinger.go b/cmd/pinger/pinger.go
@@ -1,13 +1,12 @@
 package pinger
 
 import (
-	"net/http"
 	_ "net/http/pprof" // #nosec
-	"time"
 
-	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"k8s.io/klog/v2"
+	"k8s.io/sample-controller/pkg/signals"
 
+	"github.com/kubeovn/kube-ovn/pkg/metrics"
 	"github.com/kubeovn/kube-ovn/pkg/pinger"
 	"github.com/kubeovn/kube-ovn/pkg/util"
 	"github.com/kubeovn/kube-ovn/versions"
@@ -23,20 +22,14 @@ func CmdMain() {
 	}
 	if config.Mode == "server" {
 		if config.EnableMetrics {
-			pinger.InitPingerMetrics()
-			util.InitKlogMetrics()
-
-			mux := http.NewServeMux()
-			mux.Handle("/metrics", promhttp.Handler())
 			go func() {
-				// conform to Gosec G114
-				// https://github.com/securego/gosec#available-rules
-				server := &http.Server{
-					Addr:              util.JoinHostPort("0.0.0.0", config.Port),
-					ReadHeaderTimeout: 3 * time.Second,
-					Handler:           mux,
+				pinger.InitPingerMetrics()
+				metrics.InitKlogMetrics()
+				ctx := signals.SetupSignalHandler()
+				if err := metrics.Run(ctx, nil, util.JoinHostPort("0.0.0.0", config.Port), false); err != nil {
+					util.LogFatalAndExit(err, "failed to run metrics server")
 				}
-				util.LogFatalAndExit(server.ListenAndServe(), "failed to listen and serve on %s", server.Addr)
+				<-ctx.Done()
 			}()
 		}
 

diff --git a/go.mod b/go.mod
@@ -44,7 +44,6 @@ require (
 	gopkg.in/k8snetworkplumbingwg/multus-cni.v4 v4.0.2
 	k8s.io/api v0.30.3
 	k8s.io/apimachinery v0.30.3
-	k8s.io/apiserver v0.30.3
 	k8s.io/client-go v12.0.0+incompatible
 	k8s.io/klog/v2 v2.130.1
 	k8s.io/kubectl v0.30.3
@@ -239,6 +238,7 @@ require (
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/apiextensions-apiserver v0.30.3 // indirect
+	k8s.io/apiserver v0.30.3 // indirect
 	k8s.io/cli-runtime v0.30.3 // indirect
 	k8s.io/cloud-provider v0.30.3 // indirect
 	k8s.io/cluster-bootstrap v0.30.3 // indirect