From 43faee32c113fee4e8f0e3abd451fb6d5699fdf6 Mon Sep 17 00:00:00 2001 From: jyz0309 <45495947@qq.com> Date: Tue, 23 Nov 2021 18:22:34 +0800 Subject: [PATCH 1/4] add metrics Signed-off-by: jyz0309 <45495947@qq.com> --- metrics/metrics.go | 1 + metrics/server.go | 9 +++++++++ store/helper/helper.go | 3 +++ 3 files changed, 13 insertions(+) diff --git a/metrics/metrics.go b/metrics/metrics.go index 772663f530575..f5c1faf5bb3aa 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -157,6 +157,7 @@ func RegisterMetrics() { prometheus.MustRegister(TopSQLIgnoredCounter) prometheus.MustRegister(TopSQLReportDurationHistogram) prometheus.MustRegister(TopSQLReportDataHistogram) + prometheus.MustRegister(PDApiExecutionHistogram) tikvmetrics.InitMetrics(TiDB, TiKVClient) tikvmetrics.RegisterMetrics() diff --git a/metrics/server.go b/metrics/server.go index 3bc764f6f9afa..c9d9e86a758ae 100644 --- a/metrics/server.go +++ b/metrics/server.go @@ -229,6 +229,15 @@ var ( Name: "tiflash_query_total", Help: "Counter of TiFlash queries.", }, []string{LblType, LblResult}) + + PDApiExecutionHistogram = prometheus.NewHistogram( + prometheus.HistogramOpts{ + Namespace: "tidb", + Subsystem: "server", + Name: "pd_api_execution_duration_seconds", + Help: "Bucketed histogram of all pd api execution time (s)", + Buckets: prometheus.ExponentialBuckets(0.001, 2, 20), // 1ms ~ 524s + }) ) // ExecuteErrorToLabel converts an execute error to label. diff --git a/store/helper/helper.go b/store/helper/helper.go index 3b8e122071959..13820b0a425be 100644 --- a/store/helper/helper.go +++ b/store/helper/helper.go @@ -20,6 +20,7 @@ import ( "encoding/hex" "encoding/json" "fmt" + "github.com/pingcap/tidb/metrics" "io" "math" "net/http" @@ -767,10 +768,12 @@ func (h *Helper) requestPD(method, uri string, body io.Reader, res interface{}) if err != nil { return err } + start := time.Now() resp, err := util.InternalHTTPClient().Do(req) if err != nil { return errors.Trace(err) } + metrics.PDApiExecutionHistogram.Observe(time.Since(start).Seconds()) defer func() { err = resp.Body.Close() From e426c20064c44b9b95fc9a6c882548385fa560a0 Mon Sep 17 00:00:00 2001 From: jyz0309 <45495947@qq.com> Date: Tue, 23 Nov 2021 18:25:40 +0800 Subject: [PATCH 2/4] fix lint Signed-off-by: jyz0309 <45495947@qq.com> --- store/helper/helper.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/store/helper/helper.go b/store/helper/helper.go index 13820b0a425be..dd14a85b67432 100644 --- a/store/helper/helper.go +++ b/store/helper/helper.go @@ -20,7 +20,6 @@ import ( "encoding/hex" "encoding/json" "fmt" - "github.com/pingcap/tidb/metrics" "io" "math" "net/http" @@ -35,6 +34,7 @@ import ( "github.com/pingcap/kvproto/pkg/kvrpcpb" "github.com/pingcap/log" "github.com/pingcap/tidb/kv" + "github.com/pingcap/tidb/metrics" "github.com/pingcap/tidb/parser/model" derr "github.com/pingcap/tidb/store/driver/error" "github.com/pingcap/tidb/tablecodec" From fa928af627a473ec67ba1602cb1d8cbfa2153e42 Mon Sep 17 00:00:00 2001 From: jyz0309 <45495947@qq.com> Date: Tue, 30 Nov 2021 09:54:24 +0800 Subject: [PATCH 3/4] add observe Signed-off-by: jyz0309 <45495947@qq.com> --- domain/infosync/info.go | 3 ++- metrics/server.go | 4 ++-- store/helper/helper.go | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/domain/infosync/info.go b/domain/infosync/info.go index b159100f0f92e..721a9452fd7d7 100644 --- a/domain/infosync/info.go +++ b/domain/infosync/info.go @@ -343,9 +343,10 @@ func doRequest(ctx context.Context, addrs []string, route, method string, body i if body != nil { req.Header.Set("Content-Type", "application/json") } - + start := time.Now() res, err = doRequestWithFailpoint(req) if err == nil { + metrics.PDApiExecutionHistogram.WithLabelValues("placement").Observe(time.Since(start).Seconds()) bodyBytes, err := io.ReadAll(res.Body) if err != nil { return nil, err diff --git a/metrics/server.go b/metrics/server.go index c9d9e86a758ae..68f8e8b3abf86 100644 --- a/metrics/server.go +++ b/metrics/server.go @@ -230,14 +230,14 @@ var ( Help: "Counter of TiFlash queries.", }, []string{LblType, LblResult}) - PDApiExecutionHistogram = prometheus.NewHistogram( + PDApiExecutionHistogram = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Namespace: "tidb", Subsystem: "server", Name: "pd_api_execution_duration_seconds", Help: "Bucketed histogram of all pd api execution time (s)", Buckets: prometheus.ExponentialBuckets(0.001, 2, 20), // 1ms ~ 524s - }) + }, []string{LblType}) ) // ExecuteErrorToLabel converts an execute error to label. diff --git a/store/helper/helper.go b/store/helper/helper.go index dd14a85b67432..125052d10cd75 100644 --- a/store/helper/helper.go +++ b/store/helper/helper.go @@ -773,7 +773,7 @@ func (h *Helper) requestPD(method, uri string, body io.Reader, res interface{}) if err != nil { return errors.Trace(err) } - metrics.PDApiExecutionHistogram.Observe(time.Since(start).Seconds()) + metrics.PDApiExecutionHistogram.WithLabelValues("common").Observe(time.Since(start).Seconds()) defer func() { err = resp.Body.Close() From c690c8d098ddc5266c6afeaf22624559563f7c89 Mon Sep 17 00:00:00 2001 From: jyz0309 <45495947@qq.com> Date: Wed, 1 Dec 2021 08:43:31 +0800 Subject: [PATCH 4/4] count the err case Signed-off-by: jyz0309 <45495947@qq.com> --- domain/infosync/info.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/domain/infosync/info.go b/domain/infosync/info.go index 721a9452fd7d7..29b71e8b9f276 100644 --- a/domain/infosync/info.go +++ b/domain/infosync/info.go @@ -345,8 +345,8 @@ func doRequest(ctx context.Context, addrs []string, route, method string, body i } start := time.Now() res, err = doRequestWithFailpoint(req) + metrics.PDApiExecutionHistogram.WithLabelValues("placement").Observe(time.Since(start).Seconds()) if err == nil { - metrics.PDApiExecutionHistogram.WithLabelValues("placement").Observe(time.Since(start).Seconds()) bodyBytes, err := io.ReadAll(res.Body) if err != nil { return nil, err